Lisp HTTP Client handling chunked responses
I wrote an HTTP client with ClozureCL, and this can handle chunked response.
(defparameter *remote-host* "d.hatena.ne.jp") (defparameter *remote-port* 80) (defparameter *client-stream* nil) (defmacro connect-operation (&rest body) `(with-open-socket (*client-stream* :address-family :internet :type :stream :connect :active :remote-host *remote-host* :remote-port *remote-port*) ,@body)) (defun http-read-line (&key (debug nil)) (let ((result (make-byte-array)) (stop-reading nil)) (labels ((read-http-byte () (read-byte *client-stream* nil 'eof)) (null-return (message) (when debug (format t "~A~%" message)) (setf result nil) (setf stop-reading t))) (do ((c (read-http-byte) (read-http-byte))) (stop-reading) (cond ((equal c 'eof) (null-return "Received EOF before all response received.")) ((equal c 13) (let ((next (read-byte *client-stream* nil 'eof))) ;; 10 is expected and if so, ignore 10 (if (equal next 10) (return) ;; next is not 10, received invalid packet. (null-return (format nil "Expected value is 10, but received value is ~A~%" next))))) (t (vector-push-extend c result))))) result)) (defun http-send-line (line &key (debug nil)) (when debug (format t "~A~%" line)) (princ line *client-stream*) (princ (code-char 13) *client-stream*) (princ (code-char 10) *client-stream*)) (defun read-chunked-http-response-body (&key (charset :UTF-8) (debug nil)) (format t "Received chunked response~%") (let ((response (make-byte-array))) (do () () (let ((size (decode-string-from-octets (http-read-line :debug debug) :external-format charset))) (when debug (format t "Size is ~A~%" size)) (when (equal size "0") (return)) (when debug (format t "Size is not 0, start reading response body.~%")) (dotimes (i (hex2decimal size)) (format t "~A " i) (let ((c (read-byte *client-stream* nil 'eof))) (format t "~A " c) (vector-push-extend c response))) (format t "~A" (decode-string-from-octets response :external-format charset)))))) (defun read-http-response-header (&key (debug nil)) (let ((first-line (read-line *client-stream* nil 'eof)) (response-content-length 0) (is-chunked nil)) (do ((binary-line (http-read-line :debug debug) (http-read-line :debug debug))) ((equal 0 (length binary-line)) (format t "Finished reading HTTP Response Headers!~%") (when debug (format t "is-chunked is ~A~%" is-chunked) (format t "response-content-length is ~A~%" response-content-length))) (if (null binary-line) (format t "Received unexpected packet like RST, quit.~%") (let ((line (decode-string-from-octets binary-line :external-format :UTF-8))) (let ((header (subseq line 0 (position #\: line)))) (when debug (format t "Read Response Header line :~A~%" line) (format t "Response Header: ~A~%" header)) (cond ((equal header "Content-Length") (setf response-content-length (parse-integer (subseq line (+ 1 (position #\Space line :from-end t))))) (when debug (format t "set ~A as response-content-length~%" response-content-length))) ((equal header "Transfer-Encoding") (format t "Will set is-chunked~%") (let ((value (subseq line (+ 1 (position #\Space line :from-end t))))) (when debug (format t "The value of ~A is: ~A~%" header value) (setf is-chunked (equal "chunked" value)) (format t "is-chunked is ~A~%" is-chunked))))))))) (read-chunked-http-response-body))) (defun http-get (&key (uri nil) (keep-alive-off nil) (debug nil)) (connect-operation (http-send-line (concat "GET " uri " HTTP/1.1") :debug debug) (http-send-line (concat "Host: " *remote-host*) :debug debug) (when keep-alive-off (http-send-line "Connection: close" :debug debug)) (http-send-line "" :debug debug) (force-output *client-stream*) (read-http-response-header :debug debug)))
This function uses my original functions.
Using read-line function when reading response from http server, the returned value includes 13 at the last element. But, I feel this 13 is a noise so I don't use read-line, using read-byte instead and return the value without 13 and 10.