Shammer's Philosophy

My private adversaria

Lisp HTTP Client handling chunked responses

I wrote an HTTP client with ClozureCL, and this can handle chunked response.

(defparameter *remote-host* "d.hatena.ne.jp")
(defparameter *remote-port* 80)
(defparameter *client-stream* nil)

(defmacro connect-operation (&rest body)
  `(with-open-socket (*client-stream* :address-family :internet
				      :type :stream :connect :active
				      :remote-host *remote-host*
				      :remote-port *remote-port*)
     ,@body))

(defun http-read-line (&key (debug nil))
  (let ((result (make-byte-array)) (stop-reading nil))
    (labels ((read-http-byte () (read-byte *client-stream* nil 'eof))
	     (null-return (message)
	       (when debug (format t "~A~%" message))
	       (setf result nil)
	       (setf stop-reading t)))
      (do ((c (read-http-byte) (read-http-byte)))
	  (stop-reading)
	(cond ((equal c 'eof) (null-return "Received EOF before all response received."))
	      ((equal c 13)
	       (let ((next (read-byte *client-stream* nil 'eof)))
		 ;; 10 is expected and if so, ignore 10
		 (if (equal next 10) (return)
		     ;; next is not 10, received invalid packet.
		     (null-return (format nil "Expected value is 10, but received value is ~A~%" next)))))
	      (t (vector-push-extend c result)))))
    result))

(defun http-send-line (line &key (debug nil))
  (when debug
    (format t "~A~%" line))
  (princ line *client-stream*)
  (princ (code-char 13) *client-stream*)
  (princ (code-char 10) *client-stream*))

(defun read-chunked-http-response-body (&key (charset :UTF-8) (debug nil))
  (format t "Received chunked response~%")
  (let ((response (make-byte-array)))
    (do () ()
      (let ((size (decode-string-from-octets (http-read-line :debug debug) :external-format charset)))
	(when debug
	  (format t "Size is ~A~%" size))
	(when (equal size "0")
	  (return))
	(when debug (format t "Size is not 0, start reading response body.~%"))
	(dotimes (i (hex2decimal size))
	  (format t "~A " i)
	  (let ((c (read-byte *client-stream* nil 'eof)))
	    (format t "~A " c)
	    (vector-push-extend c response)))
	(format t "~A" (decode-string-from-octets response :external-format charset))))))

(defun read-http-response-header (&key (debug nil))
  (let ((first-line (read-line *client-stream* nil 'eof))
	(response-content-length 0)
	(is-chunked nil))
    (do ((binary-line (http-read-line :debug debug) (http-read-line :debug debug)))
	((equal 0 (length binary-line))
	 (format t "Finished reading HTTP Response Headers!~%")
	 (when debug
	   (format t "is-chunked is ~A~%" is-chunked)
	   (format t "response-content-length is ~A~%" response-content-length)))
      (if (null binary-line)
	  (format t "Received unexpected packet like RST, quit.~%")
	  (let ((line (decode-string-from-octets binary-line :external-format :UTF-8)))
	    (let ((header (subseq line 0 (position #\: line))))
	      (when debug
		(format t "Read Response Header line :~A~%" line)
		(format t "Response Header: ~A~%" header))
	      (cond ((equal header "Content-Length")
		     (setf response-content-length
			   (parse-integer
			    (subseq line (+ 1 (position #\Space line :from-end t)))))
		     (when debug
		       (format t "set ~A as response-content-length~%" response-content-length)))
		    ((equal header "Transfer-Encoding")
		     (format t "Will set is-chunked~%")
		     (let ((value (subseq line (+ 1 (position #\Space line :from-end t)))))
		       (when debug
			 (format t "The value of ~A is: ~A~%" header value)
			 (setf is-chunked (equal "chunked" value))
			 (format t "is-chunked is ~A~%" is-chunked)))))))))
    (read-chunked-http-response-body)))

(defun http-get (&key (uri nil) (keep-alive-off nil) (debug nil))
  (connect-operation
   (http-send-line (concat "GET " uri " HTTP/1.1") :debug debug)
   (http-send-line (concat "Host: " *remote-host*) :debug debug)
   (when keep-alive-off
     (http-send-line "Connection: close" :debug debug))
   (http-send-line "" :debug debug)
   (force-output *client-stream*)
   (read-http-response-header :debug debug)))

This function uses my original functions.
Using read-line function when reading response from http server, the returned value includes 13 at the last element. But, I feel this 13 is a noise so I don't use read-line, using read-byte instead and return the value without 13 and 10.