diff --git a/gash/lexer.scm b/gash/lexer.scm index cf17113..4eaa2e2 100644 --- a/gash/lexer.scm +++ b/gash/lexer.scm @@ -553,10 +553,11 @@ characters read." "Convert @var{proc} from a procedure that returns a token-value pair to a procedure that returns a lexical token to be consumed by the LALR module." - (lambda (port) + (lambda (port . rest) (let ((port-location (port->port-location port))) (receive (length token) - (call-with-metered-input-port port proc) + (call-with-metered-input-port port (lambda (mp) + (apply proc mp rest))) (match token ((category . value) (make-lexical-token @@ -570,7 +571,15 @@ module." (get-token->get-lexical-token get-operator)) (define get-word-lexical-token - (get-token->get-lexical-token get-word)) + ;; The is a roundabout way to avoid putting more than one character + ;; on the 'unget-char' buffer for regular ports. Mes only allows + ;; one. Here, we delay adding a character until after we have + ;; initialized a soft port, which has an unbounded buffer. + (let ((get-word* (lambda* (port #:key start) + (when start + (unget-char port start)) + (get-word port)))) + (get-token->get-lexical-token get-word*))) (define (get-newline-lexical-token port) "Get a newline as a lexical token to be consumed by the LALR module." @@ -602,8 +611,7 @@ is a newline (or EOF)." (#\newline (get-newline-lexical-token port)) (#\\ (match (next-char port) (#\newline (loop (next-char port))) - (_ (unget-char port #\\) - (get-word-lexical-token port)))) + (_ (get-word-lexical-token port #:start #\\)))) (_ (get-word-lexical-token port)))))