http.lua 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382
  1. -----------------------------------------------------------------------------
  2. -- HTTP/1.1 client support for the Lua language.
  3. -- LuaSocket toolkit.
  4. -- Author: Diego Nehab
  5. -----------------------------------------------------------------------------
  6. -----------------------------------------------------------------------------
  7. -- Declare module and import dependencies
  8. -------------------------------------------------------------------------------
  9. local socket = require("socket")
  10. local url = require("socket.url")
  11. local ltn12 = require("ltn12")
  12. local mime = require("mime")
  13. local string = require("string")
  14. local headers = require("socket.headers")
  15. local base = _G
  16. local table = require("table")
  17. socket.http = {}
  18. local _M = socket.http
  19. -----------------------------------------------------------------------------
  20. -- Program constants
  21. -----------------------------------------------------------------------------
  22. -- connection timeout in seconds
  23. _M.TIMEOUT = 60
  24. -- user agent field sent in request
  25. _M.USERAGENT = socket._VERSION
  26. -- supported schemes
  27. local SCHEMES = { ["http"] = true }
  28. -- default port for document retrieval
  29. local PORT = 80
  30. -----------------------------------------------------------------------------
  31. -- Reads MIME headers from a connection, unfolding where needed
  32. -----------------------------------------------------------------------------
  33. local function receiveheaders(sock, headers)
  34. local line, name, value, err
  35. headers = headers or {}
  36. -- get first line
  37. line, err = sock:receive()
  38. if err then return nil, err end
  39. -- headers go until a blank line is found
  40. while line ~= "" do
  41. -- get field-name and value
  42. name, value = socket.skip(2, string.find(line, "^(.-):%s*(.*)"))
  43. if not (name and value) then return nil, "malformed reponse headers" end
  44. name = string.lower(name)
  45. -- get next line (value might be folded)
  46. line, err = sock:receive()
  47. if err then return nil, err end
  48. -- unfold any folded values
  49. while string.find(line, "^%s") do
  50. value = value .. line
  51. line = sock:receive()
  52. if err then return nil, err end
  53. end
  54. -- save pair in table
  55. if headers[name] then headers[name] = headers[name] .. ", " .. value
  56. else headers[name] = value end
  57. end
  58. return headers
  59. end
  60. -----------------------------------------------------------------------------
  61. -- Extra sources and sinks
  62. -----------------------------------------------------------------------------
  63. socket.sourcet["http-chunked"] = function(sock, headers)
  64. return base.setmetatable({
  65. getfd = function() return sock:getfd() end,
  66. dirty = function() return sock:dirty() end
  67. }, {
  68. __call = function()
  69. -- get chunk size, skip extention
  70. local line, err = sock:receive()
  71. if err then return nil, err end
  72. local size = base.tonumber(string.gsub(line, ";.*", ""), 16)
  73. if not size then return nil, "invalid chunk size" end
  74. -- was it the last chunk?
  75. if size > 0 then
  76. -- if not, get chunk and skip terminating CRLF
  77. local chunk, err, part = sock:receive(size)
  78. if chunk then sock:receive() end
  79. return chunk, err
  80. else
  81. -- if it was, read trailers into headers table
  82. headers, err = receiveheaders(sock, headers)
  83. if not headers then return nil, err end
  84. end
  85. end
  86. })
  87. end
  88. socket.sinkt["http-chunked"] = function(sock)
  89. return base.setmetatable({
  90. getfd = function() return sock:getfd() end,
  91. dirty = function() return sock:dirty() end
  92. }, {
  93. __call = function(self, chunk, err)
  94. if not chunk then return sock:send("0\r\n\r\n") end
  95. local size = string.format("%X\r\n", string.len(chunk))
  96. return sock:send(size .. chunk .. "\r\n")
  97. end
  98. })
  99. end
  100. -----------------------------------------------------------------------------
  101. -- Low level HTTP API
  102. -----------------------------------------------------------------------------
  103. local metat = { __index = {} }
  104. function _M.open(host, port, create)
  105. -- create socket with user connect function, or with default
  106. local c = socket.try((create or socket.tcp)())
  107. local h = base.setmetatable({ c = c }, metat)
  108. -- create finalized try
  109. h.try = socket.newtry(function() h:close() end)
  110. -- set timeout before connecting
  111. h.try(c:settimeout(_M.TIMEOUT))
  112. h.try(c:connect(host, port or PORT))
  113. -- here everything worked
  114. return h
  115. end
  116. function metat.__index:sendrequestline(method, uri)
  117. local reqline = string.format("%s %s HTTP/1.1\r\n", method or "GET", uri)
  118. return self.try(self.c:send(reqline))
  119. end
  120. function metat.__index:sendheaders(tosend)
  121. local canonic = headers.canonic
  122. local h = "\r\n"
  123. for f, v in base.pairs(tosend) do
  124. h = (canonic[f] or f) .. ": " .. v .. "\r\n" .. h
  125. end
  126. self.try(self.c:send(h))
  127. return 1
  128. end
  129. function metat.__index:sendbody(headers, source, step)
  130. source = source or ltn12.source.empty()
  131. step = step or ltn12.pump.step
  132. -- if we don't know the size in advance, send chunked and hope for the best
  133. local mode = "http-chunked"
  134. if headers["content-length"] then mode = "keep-open" end
  135. return self.try(ltn12.pump.all(source, socket.sink(mode, self.c), step))
  136. end
  137. function metat.__index:receivestatusline()
  138. local status = self.try(self.c:receive(5))
  139. -- identify HTTP/0.9 responses, which do not contain a status line
  140. -- this is just a heuristic, but is what the RFC recommends
  141. if status ~= "HTTP/" then return nil, status end
  142. -- otherwise proceed reading a status line
  143. status = self.try(self.c:receive("*l", status))
  144. local code = socket.skip(2, string.find(status, "HTTP/%d*%.%d* (%d%d%d)"))
  145. return self.try(base.tonumber(code), status)
  146. end
  147. function metat.__index:receiveheaders()
  148. return self.try(receiveheaders(self.c))
  149. end
  150. function metat.__index:receivebody(headers, sink, step)
  151. sink = sink or ltn12.sink.null()
  152. step = step or ltn12.pump.step
  153. local length = base.tonumber(headers["content-length"])
  154. local t = headers["transfer-encoding"] -- shortcut
  155. local mode = "default" -- connection close
  156. if t and t ~= "identity" then mode = "http-chunked"
  157. elseif base.tonumber(headers["content-length"]) then mode = "by-length" end
  158. return self.try(ltn12.pump.all(socket.source(mode, self.c, length),
  159. sink, step))
  160. end
  161. function metat.__index:receive09body(status, sink, step)
  162. local source = ltn12.source.rewind(socket.source("until-closed", self.c))
  163. source(status)
  164. return self.try(ltn12.pump.all(source, sink, step))
  165. end
  166. function metat.__index:close()
  167. return self.c:close()
  168. end
  169. -----------------------------------------------------------------------------
  170. -- High level HTTP API
  171. -----------------------------------------------------------------------------
  172. local function adjusturi(reqt)
  173. local u = reqt
  174. -- if there is a proxy, we need the full url. otherwise, just a part.
  175. if not reqt.proxy and not _M.PROXY then
  176. u = {
  177. path = socket.try(reqt.path, "invalid path 'nil'"),
  178. params = reqt.params,
  179. query = reqt.query,
  180. fragment = reqt.fragment
  181. }
  182. end
  183. return url.build(u)
  184. end
  185. local function adjustproxy(reqt)
  186. local proxy = reqt.proxy or _M.PROXY
  187. if proxy then
  188. proxy = url.parse(proxy)
  189. return proxy.host, proxy.port or 3128
  190. else
  191. return reqt.host, reqt.port
  192. end
  193. end
  194. local function adjustheaders(reqt)
  195. -- default headers
  196. local host = string.gsub(reqt.authority, "^.-@", "")
  197. local lower = {
  198. ["user-agent"] = _M.USERAGENT,
  199. ["host"] = host,
  200. ["connection"] = "close, TE",
  201. ["te"] = "trailers"
  202. }
  203. -- if we have authentication information, pass it along
  204. if reqt.user and reqt.password then
  205. lower["authorization"] =
  206. "Basic " .. (mime.b64(reqt.user .. ":" ..
  207. url.unescape(reqt.password)))
  208. end
  209. -- if we have proxy authentication information, pass it along
  210. local proxy = reqt.proxy or _M.PROXY
  211. if proxy then
  212. proxy = url.parse(proxy)
  213. if proxy.user and proxy.password then
  214. lower["proxy-authorization"] =
  215. "Basic " .. (mime.b64(proxy.user .. ":" .. proxy.password))
  216. end
  217. end
  218. -- override with user headers
  219. for i,v in base.pairs(reqt.headers or lower) do
  220. lower[string.lower(i)] = v
  221. end
  222. return lower
  223. end
  224. -- default url parts
  225. local default = {
  226. host = "",
  227. port = PORT,
  228. path ="/",
  229. scheme = "http"
  230. }
  231. local function adjustrequest(reqt)
  232. -- parse url if provided
  233. local nreqt = reqt.url and url.parse(reqt.url, default) or {}
  234. -- explicit components override url
  235. for i,v in base.pairs(reqt) do nreqt[i] = v end
  236. if nreqt.port == "" then nreqt.port = PORT end
  237. if not (nreqt.host and nreqt.host ~= "") then
  238. socket.try(nil, "invalid host '" .. base.tostring(nreqt.host) .. "'")
  239. end
  240. -- compute uri if user hasn't overriden
  241. nreqt.uri = reqt.uri or adjusturi(nreqt)
  242. -- adjust headers in request
  243. nreqt.headers = adjustheaders(nreqt)
  244. -- ajust host and port if there is a proxy
  245. nreqt.host, nreqt.port = adjustproxy(nreqt)
  246. return nreqt
  247. end
  248. local function shouldredirect(reqt, code, headers)
  249. local location = headers.location
  250. if not location then return false end
  251. location = string.gsub(location, "%s", "")
  252. if location == "" then return false end
  253. local scheme = string.match(location, "^([%w][%w%+%-%.]*)%:")
  254. if scheme and not SCHEMES[scheme] then return false end
  255. return (reqt.redirect ~= false) and
  256. (code == 301 or code == 302 or code == 303 or code == 307) and
  257. (not reqt.method or reqt.method == "GET" or reqt.method == "HEAD")
  258. and (not reqt.nredirects or reqt.nredirects < 5)
  259. end
  260. local function shouldreceivebody(reqt, code)
  261. if reqt.method == "HEAD" then return nil end
  262. if code == 204 or code == 304 then return nil end
  263. if code >= 100 and code < 200 then return nil end
  264. return 1
  265. end
  266. -- forward declarations
  267. local trequest, tredirect
  268. --[[local]] function tredirect(reqt, location)
  269. local result, code, headers, status = trequest {
  270. -- the RFC says the redirect URL has to be absolute, but some
  271. -- servers do not respect that
  272. url = url.absolute(reqt.url, location),
  273. source = reqt.source,
  274. sink = reqt.sink,
  275. headers = reqt.headers,
  276. proxy = reqt.proxy,
  277. nredirects = (reqt.nredirects or 0) + 1,
  278. create = reqt.create
  279. }
  280. -- pass location header back as a hint we redirected
  281. headers = headers or {}
  282. headers.location = headers.location or location
  283. return result, code, headers, status
  284. end
  285. --[[local]] function trequest(reqt)
  286. -- we loop until we get what we want, or
  287. -- until we are sure there is no way to get it
  288. local nreqt = adjustrequest(reqt)
  289. local h = _M.open(nreqt.host, nreqt.port, nreqt.create)
  290. -- send request line and headers
  291. h:sendrequestline(nreqt.method, nreqt.uri)
  292. h:sendheaders(nreqt.headers)
  293. -- if there is a body, send it
  294. if nreqt.source then
  295. h:sendbody(nreqt.headers, nreqt.source, nreqt.step)
  296. end
  297. local code, status = h:receivestatusline()
  298. -- if it is an HTTP/0.9 server, simply get the body and we are done
  299. if not code then
  300. h:receive09body(status, nreqt.sink, nreqt.step)
  301. return 1, 200
  302. end
  303. local headers
  304. -- ignore any 100-continue messages
  305. while code == 100 do
  306. headers = h:receiveheaders()
  307. code, status = h:receivestatusline()
  308. end
  309. headers = h:receiveheaders()
  310. -- at this point we should have a honest reply from the server
  311. -- we can't redirect if we already used the source, so we report the error
  312. if shouldredirect(nreqt, code, headers) and not nreqt.source then
  313. h:close()
  314. return tredirect(reqt, headers.location)
  315. end
  316. -- here we are finally done
  317. if shouldreceivebody(nreqt, code) then
  318. h:receivebody(headers, nreqt.sink, nreqt.step)
  319. end
  320. h:close()
  321. return 1, code, headers, status
  322. end
  323. -- turns an url and a body into a generic request
  324. local function genericform(u, b)
  325. local t = {}
  326. local reqt = {
  327. url = u,
  328. sink = ltn12.sink.table(t),
  329. target = t
  330. }
  331. if b then
  332. reqt.source = ltn12.source.string(b)
  333. reqt.headers = {
  334. ["content-length"] = string.len(b),
  335. ["content-type"] = "application/x-www-form-urlencoded"
  336. }
  337. reqt.method = "POST"
  338. end
  339. return reqt
  340. end
  341. _M.genericform = genericform
  342. local function srequest(u, b)
  343. local reqt = genericform(u, b)
  344. local _, code, headers, status = trequest(reqt)
  345. return table.concat(reqt.target), code, headers, status
  346. end
  347. _M.request = socket.protect(function(reqt, body)
  348. if base.type(reqt) == "string" then return srequest(reqt, body)
  349. else return trequest(reqt) end
  350. end)
  351. return _M