(defmacro with-tokens ((i string predicate) &body body)
  "Tokenise string and execute body with i bound to each token.
The beginning of a token is indicated by predicate, which should
be a function of a single character."
  (with-gensyms (next current end my-string my-predicate)
    `(let* ((,my-string ,string)
            (,my-predicate ,predicate)
            (,current 0)
            (,end (length ,my-string)))
       (loop while (< ,current ,end) do
            (let ((,next ,current))
              (loop until (or (= ,next ,end)
                              (funcall ,my-predicate
                                       (aref ,my-string ,next))) do
                   (incf ,next))
              (let ((,i (subseq ,my-string ,current ,next)))
                (unless (equal ,i "")
                  ,@body)
                (setf ,current ,next)
                (loop until (or (= ,next ,end)
                                (not (funcall ,my-predicate
                                              (aref ,my-string ,next)))) do
                     (incf ,next))
                (let ((,i (subseq ,my-string ,current ,next)))
                  (unless (equal ,i "")
                    ,@body))
                (setf ,current ,next)))))))

(defun untokenise (list)
  "Un-tokenise a list into a string, with tokens separated by spaces"
  (string-trim '(#\Space) (format nil "~{~A ~}" list)))

(defun make-simple-tokeniser (chars)
  "Make a simple tokeniser that splits on the given set of characters"
  (setf chars (mapcar (lambda (x) (coerce x 'character)) chars))
  (lambda (char)
    (member char chars)))

(defun make-nesting-tokeniser (chars) ;; anything between { and } is a single token
  "Make a tokeniser that splits on the given characters, and on balanced curly braces"
  (setf chars (mapcar (lambda (x) (coerce x 'character)) chars))
  (let ((nest-level 0))
    (lambda (char)
      (when (eql char #\{)
        (incf nest-level))
      (when (eql char #\})
        (decf nest-level))
      (and (= nest-level 0) (member char chars)))))