From 3c2fba9a9ffb4aac0378010c210c2457eeefa559 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marcel=20Fabian=20Kr=C3=BCger?= <zauguin@gmail.com>
Date: Thu, 13 Aug 2020 22:24:13 +0200
Subject: [PATCH] Adapt to new LuaMetaTeX versions and allow CB use

---
 luametalatex-back-pdf.lua       | 315 ++++++++++++++++++--------------
 luametalatex-basecallbacks.lua  | 111 ++++++++---
 luametalatex-baseregisters.lua  |  78 ++++----
 luametalatex-build-bytecode.lua |   2 +-
 luametalatex-callbacks.lua      |  53 +++++-
 luametalatex-dir-registers.lua  |  12 +-
 luametalatex-firstcode.lua      | 234 +++++++++++++++++++-----
 luametalatex-font-resolve.lua   |   6 +-
 luametalatex-lateinit.lua       |  12 +-
 luametalatex-ltexpl-hook.tex    |   4 +
 luametalatex-microtype.sty      |   8 +-
 luametalatex-pdf.lua            |   1 +
 12 files changed, 567 insertions(+), 269 deletions(-)

diff --git a/luametalatex-back-pdf.lua b/luametalatex-back-pdf.lua
index cb792b4..bd4120d 100644
--- a/luametalatex-back-pdf.lua
+++ b/luametalatex-back-pdf.lua
@@ -1,6 +1,16 @@
+local scan_int = token.scan_integer
+local scan_token = token.scan_token
+local scan_keyword = token.scan_keyword
+local scan_string = token.scan_string
+local scan_word = token.scan_word
+local scan_dimen = token.scan_dimen
+local scan_box = token.scan_box
+token.scan_list = scan_box -- They are equal if no parameter is present
+
 local pdf = pdf
 local pdfvariable = pdf.variable
 
+local callbacks = require'luametalatex-callbacks'
 local writer = require'luametalatex-nodewriter'
 local newpdf = require'luametalatex-pdf'
 local nametree = require'luametalatex-pdf-nametree'
@@ -36,7 +46,6 @@ local colorstacks = {{
     default = "0 g 0 G",
     page_stack = {"0 g 0 G"},
   }}
-token.scan_list = token.scan_box -- They are equal if no parameter is present
 local spacer_cmd = token.command_id'spacer'
 local function get_pfile()
   if not pfile then
@@ -71,7 +80,7 @@ token.luacmd("shipout", function()
   local total_voffset, total_hoffset = tex.voffset + pdfvariable.vorigin, tex.hoffset + pdfvariable.horigin
   local voff = node.new'kern'
   voff.kern = total_voffset
-  voff.next = token.scan_list()
+  voff.next = scan_box()
   voff.next.shift = total_hoffset
   local list = node.direct.tonode(node.direct.vpack(node.direct.todirect(voff)))
   local pageheight, pagewidth = tex.pageheight, tex.pagewidth
@@ -87,7 +96,7 @@ token.luacmd("shipout", function()
   pfile:indirect(page, string.format([[<</Type/Page/Parent %i 0 R/Contents %i 0 R/MediaBox[0 %i %i %i]/Resources%s%s%s%s>>]], parent, content, -math.ceil(to_bp(list.depth)), math.ceil(to_bp(list.width)), math.ceil(to_bp(list.height)), resources(pdfvariable.pageresources .. pdf.pageresources), annots, pdfvariable.pageattr, pdf.pageattributes))
   node.flush_list(list)
   token.put_next(reset_deadcycles)
-  token.scan_token()
+  scan_token()
 end, 'force', 'protected')
 
 local infodir = ""
@@ -153,7 +162,10 @@ local function nodefont_newindex(t, k, v)
   return rawset(t, k, v)
 end
 
-callback.register("stop_run", function()
+function callbacks.stop_run()
+  local user_callback = callbacks.stop_run
+  if user_callback then user_callback() end
+
   if not pfile then
     return
   end
@@ -234,19 +246,21 @@ callback.register("stop_run", function()
   texio.write_nl("  " .. table.concat(nodestat, ', '))
   texio.write_nl(string.format("Output written on %s (%d pages, %d bytes).", pdfname, pages, size))
   texio.write_nl(string.format("Transcript written on %s.\n", status.log_name))
-end, "Finish PDF file")
+end
+callbacks.__freeze('stop_run', true)
+
 token.luacmd("pdfvariable", function()
   for _, n in ipairs(pdf.variable_names) do
-    if token.scan_keyword(n) then
+    if scan_keyword(n) then
       return token.put_next(token.create('pdfvariable  ' .. n))
     end
   end
   -- The following error message gobbles the next word as a side effect.
   -- This is intentional to make error-recovery easier.
   --[[
-  error(string.format("Unknown PDF variable %s", token.scan_word()))
+  error(string.format("Unknown PDF variable %s", scan_word()))
   ]] -- Delay the error to ensure luatex85.sty compatibility
-  texio.write_nl(string.format("Unknown PDF variable %s", token.scan_word()))
+  texio.write_nl(string.format("Unknown PDF variable %s", scan_word()))
   tex.sprint"\\unexpanded{\\undefinedpdfvariable}"
 end)
 
@@ -534,7 +548,7 @@ local colorstack_whatsit = declare_whatsit('pdf_colorstack', function(prop, p, n
   pdf.write(colorstack.mode, stack[#stack], x, y, p)
 end)
 local function write_colorstack()
-  local idx = token.scan_int()
+  local idx = scan_int()
   local colorstack = colorstacks[idx + 1]
   if not colorstack then
     tex.error('Undefined colorstack', {"The requested colorstack is not initialized. \z
@@ -542,10 +556,10 @@ local function write_colorstack()
         that you specified the wrong index. I will continue with colorstack 0."})
     colorstack = colorstacks[1]
   end
-  local action = token.scan_keyword'pop' and 'pop'
-              or token.scan_keyword'set' and 'set'
-              or token.scan_keyword'current' and 'current'
-              or token.scan_keyword'push' and 'push'
+  local action = scan_keyword'pop' and 'pop'
+              or scan_keyword'set' and 'set'
+              or scan_keyword'current' and 'current'
+              or scan_keyword'push' and 'push'
   if not action then
     tex.error('Missing action specifier for colorstack', {
         "I don't know what you want to do with this colorstack. I would have expected pop/set/current or push here. \z
@@ -554,7 +568,7 @@ local function write_colorstack()
   end
   local text
   if action == "push" or "set" then
-    text = token.scan_string()
+    text = scan_string()
     -- text = token.to_string(token.scan_tokenlist()) -- Attention! This should never be executed in an expand-only context
   end
   local whatsit = node.new(whatsit_id, colorstack_whatsit)
@@ -568,97 +582,103 @@ end
 local function scan_action()
   local action_type
   
-  if token.scan_keyword'user' then
-    return {action_type = 3, data = token.scan_string()}
-  elseif token.scan_keyword'thread' then
+  if scan_keyword'user' then
+    return {action_type = 3, data = scan_string()}
+  elseif scan_keyword'thread' then
     error[[FIXME: Unsupported]] -- TODO
-  elseif token.scan_keyword'goto' then
+  elseif scan_keyword'goto' then
     action_type = 1
   else
     error[[Unsupported action]]
   end
   local action = {
     action_type = action_type,
-    file = token.scan_keyword'file' and token.scan_string(),
+    file = scan_keyword'file' and scan_string(),
   }
-  if token.scan_keyword'page' then
+  if scan_keyword'page' then
     assert(action_type == 1)
     action_type = 0
-    local page = token.scan_int()
+    action.action_type = 0
+    local page = scan_int()
     if page <= 0 then
       error[[page must be positive in action specification]]
     end
     action.page = page
-    action.tokens = token.scan_string()
-  elseif token.scan_keyword'num' then
+    action.tokens = scan_string()
+  elseif scan_keyword'num' then
     if action.file and action_type == 1 then
       error[[num style GoTo actions must be internal]]
     end
-    action.id = token.scan_int()
+    action.id = scan_int()
     if action.id <= 0 then
       error[[id must be positive]]
     end
-  elseif token.scan_keyword'name' then
-    action.id = token.scan_string()
+  elseif scan_keyword'name' then
+    action.id = scan_string()
   else
     error[[Unsupported id type]]
   end
-  action.new_window = token.scan_keyword'newwindow' and 1
-                   or token.scan_keyword'nonewwindow' and 2
+  action.new_window = scan_keyword'newwindow' and 1
+                   or scan_keyword'nonewwindow' and 2
   if action.new_window and not action.file then
     error[[newwindow is only supported for external files]]
   end
   return action
 end
 local function scan_literal_mode()
-  return token.scan_keyword"direct" and "direct"
-      or token.scan_keyword"page" and "page"
-      or token.scan_keyword"text" and "text"
-      or token.scan_keyword"direct" and "direct"
-      or token.scan_keyword"raw" and "raw"
+  return scan_keyword"direct" and "direct"
+      or scan_keyword"page" and "page"
+      or scan_keyword"text" and "text"
+      or scan_keyword"direct" and "direct"
+      or scan_keyword"raw" and "raw"
       or "origin"
 end
 local function maybe_gobble_cmd(cmd)
-  local t = token.scan_token()
+  local t = scan_token()
   if t.command ~= cmd then
     token.put_next(t)
   end
 end
 token.luacmd("pdffeedback", function()
-  if token.scan_keyword"colorstackinit" then
-    local page = token.scan_keyword'page'
-              or (token.scan_keyword'nopage' and false) -- If you want to pass "page" as mode
+  if scan_keyword"colorstackinit" then
+    local page = scan_keyword'page'
+              or (scan_keyword'nopage' and false) -- If you want to pass "page" as mode
     local mode = scan_literal_mode()
-    local default = token.scan_string()
+    local default = scan_string()
     tex.sprint(tostring(pdf.newcolorstack(default, mode, page)))
-  elseif token.scan_keyword"creationdate" then
+  elseif scan_keyword"creationdate" then
     tex.sprint(creationdate)
-  elseif token.scan_keyword"lastannot" then
+  elseif scan_keyword"lastannot" then
     tex.sprint(tostring(lastannot))
-  elseif token.scan_keyword"lastobj" then
+  elseif scan_keyword"lastobj" then
     tex.sprint(tostring(lastobj))
   else
   -- The following error message gobbles the next word as a side effect.
   -- This is intentional to make error-recovery easier.
-    error(string.format("Unknown PDF feedback %s", token.scan_word()))
+    error(string.format("Unknown PDF feedback %s", scan_word()))
   end
 end)
-token.luacmd("pdfextension", function(_, imm)
-  if token.scan_keyword"colorstack" then
+token.luacmd("pdfextension", function(_, immediate)
+  if immediate == "value" then return end
+  if immediate and immediate & 0x7 ~= 0 then
+    immediate = immediate & 0x8
+    tex.error("Unexpected prefix", "You used \\pdfextension with a prefix that doesn't belong there. I will ignore it for now.")
+  end
+  if scan_keyword"colorstack" then
     write_colorstack()
-  elseif token.scan_keyword"literal" then
+  elseif scan_keyword"literal" then
     local mode = scan_literal_mode()
-    local literal = token.scan_string()
+    local literal = scan_string()
     local whatsit = node.new(whatsit_id, literal_whatsit)
     node.setproperty(whatsit, {
         mode = mode,
         data = literal,
       })
     node.write(whatsit)
-  elseif token.scan_keyword"startlink" then
+  elseif scan_keyword"startlink" then
     local pfile = get_pfile()
     local whatsit = node.new(whatsit_id, start_link_whatsit)
-    local attr = token.scan_keyword'attr' and token.scan_string() or ''
+    local attr = scan_keyword'attr' and scan_string() or ''
     local action = scan_action()
     local objnum = pfile:getobj()
     lastannot = num
@@ -668,27 +688,27 @@ token.luacmd("pdfextension", function(_, imm)
         objnum = objnum,
       })
     node.write(whatsit)
-  elseif token.scan_keyword"endlink" then
+  elseif scan_keyword"endlink" then
     local whatsit = node.new(whatsit_id, end_link_whatsit)
     node.write(whatsit)
-  elseif token.scan_keyword"save" then
+  elseif scan_keyword"save" then
     local whatsit = node.new(whatsit_id, save_whatsit)
     node.write(whatsit)
-  elseif token.scan_keyword"setmatrix" then
-    local matrix = token.scan_string()
+  elseif scan_keyword"setmatrix" then
+    local matrix = scan_string()
     local whatsit = node.new(whatsit_id, setmatrix_whatsit)
     node.setproperty(whatsit, {
         data = matrix,
       })
     node.write(whatsit)
-  elseif token.scan_keyword"restore" then
+  elseif scan_keyword"restore" then
     local whatsit = node.new(whatsit_id, restore_whatsit)
     node.write(whatsit)
-  elseif token.scan_keyword"info" then
-    infodir = infodir .. token.scan_string()
-  elseif token.scan_keyword"catalog" then
-    catalogdir = catalogdir .. ' ' .. token.scan_string()
-    if token.scan_keyword'openaction' then
+  elseif scan_keyword"info" then
+    infodir = infodir .. scan_string()
+  elseif scan_keyword"catalog" then
+    catalogdir = catalogdir .. ' ' .. scan_string()
+    if scan_keyword'openaction' then
       if catalog_openaction then
         tex.error("Duplicate openaction", {"Only one use of \\pdfextension catalog is allowed to \z
             have an openaction."})
@@ -697,19 +717,19 @@ token.luacmd("pdfextension", function(_, imm)
         catalog_openaction = get_action_attr(get_pfile(), action)
       end
     end
-  elseif token.scan_keyword"names" then
-    namesdir = namesdir .. ' ' .. token.scan_string()
-  elseif token.scan_keyword"obj" then
+  elseif scan_keyword"names" then
+    namesdir = namesdir .. ' ' .. scan_string()
+  elseif scan_keyword"obj" then
     local pfile = get_pfile()
-    if token.scan_keyword"reserveobjnum" then
+    if scan_keyword"reserveobjnum" then
       lastobj = pfile:getobj()
     else
-      local num = token.scan_keyword'useobjnum' and token.scan_int() or pfile:getobj()
+      local num = scan_keyword'useobjnum' and scan_int() or pfile:getobj()
       lastobj = num
-      local attr = token.scan_keyword'stream' and (token.scan_keyword'attr' and token.scan_string() or '')
-      local isfile = token.scan_keyword'file'
-      local content = token.scan_string()
-      if imm == 'immediate' then
+      local attr = scan_keyword'stream' and (scan_keyword'attr' and scan_string() or '')
+      local isfile = scan_keyword'file'
+      local content = scan_string()
+      if immediate == 8 then
         if attr then
           pfile:stream(num, attr, content, isfile)
         else
@@ -723,43 +743,43 @@ token.luacmd("pdfextension", function(_, imm)
         end
       end
     end
-  elseif token.scan_keyword"refobj" then
-    local num = token.scan_int()
+  elseif scan_keyword"refobj" then
+    local num = scan_int()
     local whatsit = node.new(whatsit_id, refobj_whatsit)
     node.setproperty(whatsit, {
         obj = num,
       })
     node.write(whatsit)
-  elseif token.scan_keyword"outline" then
+  elseif scan_keyword"outline" then
     local pfile = get_pfile()
-    local attr = token.scan_keyword'attr' and token.scan_string() or ''
+    local attr = scan_keyword'attr' and scan_string() or ''
     local action
-    if token.scan_keyword"useobjnum" then
-      action = token.scan_int()
+    if scan_keyword"useobjnum" then
+      action = scan_int()
     else
       local actionobj = scan_action()
       action = pfile:indirect(nil, get_action_attr(pfile, actionobj))
     end
     local outline = get_outline()
-    if token.scan_keyword'level' then
-      local level = token.scan_int()
-      local open = token.scan_keyword'open'
-      local title = token.scan_string()
+    if scan_keyword'level' then
+      local level = scan_int()
+      local open = scan_keyword'open'
+      local title = scan_string()
       outline:add(pdf_text(title), action, level, open, attr)
     else
-      local count = token.scan_keyword'count' and token.scan_int() or 0
-      local title = token.scan_string()
+      local count = scan_keyword'count' and scan_int() or 0
+      local title = scan_string()
       outline:add_legacy(pdf_text(title), action, count, attr)
     end
-  elseif token.scan_keyword"dest" then
+  elseif scan_keyword"dest" then
     local id
-    if token.scan_keyword'num' then
-      id = token.scan_int()
+    if scan_keyword'num' then
+      id = scan_int()
       if id <= 0 then
         error[[id must be positive]]
       end
-    elseif token.scan_keyword'name' then
-      id = token.scan_string()
+    elseif scan_keyword'name' then
+      id = scan_string()
     else
       error[[Unsupported id type]]
     end
@@ -768,54 +788,54 @@ token.luacmd("pdfextension", function(_, imm)
       dest_id = id,
     }
     node.setproperty(whatsit, prop)
-    if token.scan_keyword'xyz' then
+    if scan_keyword'xyz' then
       prop.dest_type = 'xyz'
-      prop.xyz_zoom = token.scan_keyword'zoom' and token.scan_int()
+      prop.xyz_zoom = scan_keyword'zoom' and scan_int()
       maybe_gobble_cmd(spacer_cmd)
-    elseif token.scan_keyword'fitr' then
+    elseif scan_keyword'fitr' then
       prop.dest_type = 'fitr'
       maybe_gobble_cmd(spacer_cmd)
       while true do
-        if token.scan_keyword'width' then
-          prop.width = token.scan_dimen()
-        elseif token.scan_keyword'height' then
-          prop.height = token.scan_dimen()
-        elseif token.scan_keyword'depth' then
-          prop.depth = token.scan_dimen()
+        if scan_keyword'width' then
+          prop.width = scan_dimen()
+        elseif scan_keyword'height' then
+          prop.height = scan_dimen()
+        elseif scan_keyword'depth' then
+          prop.depth = scan_dimen()
         else
           break
         end
       end
-    elseif token.scan_keyword'fitbh' then
+    elseif scan_keyword'fitbh' then
       prop.dest_type = 'fitbh'
       maybe_gobble_cmd(spacer_cmd)
-    elseif token.scan_keyword'fitbv' then
+    elseif scan_keyword'fitbv' then
       prop.dest_type = 'fitbv'
       maybe_gobble_cmd(spacer_cmd)
-    elseif token.scan_keyword'fitb' then
+    elseif scan_keyword'fitb' then
       prop.dest_type = 'fitb'
       maybe_gobble_cmd(spacer_cmd)
-    elseif token.scan_keyword'fith' then
+    elseif scan_keyword'fith' then
       prop.dest_type = 'fith'
       maybe_gobble_cmd(spacer_cmd)
-    elseif token.scan_keyword'fitv' then
+    elseif scan_keyword'fitv' then
       prop.dest_type = 'fitv'
       maybe_gobble_cmd(spacer_cmd)
-    elseif token.scan_keyword'fit' then
+    elseif scan_keyword'fit' then
       prop.dest_type = 'fit'
       maybe_gobble_cmd(spacer_cmd)
     else
       error[[Unsupported dest type]]
     end
     node.write(whatsit)
-  elseif token.scan_keyword'mapline' then
-    fontmap.mapline(token.scan_string())
+  elseif scan_keyword'mapline' then
+    fontmap.mapline(scan_string())
   else
   -- The following error message gobbles the next word as a side effect.
   -- This is intentional to make error-recovery easier.
-    error(string.format("Unknown PDF extension %s", token.scan_word()))
+    error(string.format("Unknown PDF extension %s", scan_word()))
   end
-end, "protected")
+end, "value")
 local imglib = require'luametalatex-pdf-image'
 local imglib_node = imglib.node
 local imglib_write = imglib.write
@@ -832,19 +852,25 @@ local lastimage = -1
 local lastimagepages = -1
 
 -- These are very minimal right now but LaTeX isn't using the scaling etc. stuff anyway.
-token.luacmd("saveimageresource", function(imm)
-  local attr = token.scan_keyword'attr' and token.scan_string() or nil
-  local page = token.scan_keyword'page' and token.scan_int() or nil
-  local userpassword = token.scan_keyword'userpassword' and token.scan_string() or nil
-  local ownerpassword = token.scan_keyword'ownerpassword' and token.scan_string() or nil
-  -- local colorspace = token.scan_keyword'colorspace' and token.scan_int() or nil -- Doesn't make sense for PDF
-  local pagebox = token.scan_keyword'mediabox' and 'media'
-               or token.scan_keyword'cropbox' and 'crop'
-               or token.scan_keyword'bleedbox' and 'bleed'
-               or token.scan_keyword'trimbox' and 'trim'
-               or token.scan_keyword'artbox' and 'art'
+token.luacmd("saveimageresource", function(_, immediate)
+  if immediate == "value" then return end
+  if immediate and immediate & 0x7 ~= 0 then
+    print(immediate)
+    immediate = immediate & 0x8
+    tex.error("Unexpected prefix", "You used \\saveimageresource with a prefix that doesn't belong there. I will ignore it for now.")
+  end
+  local attr = scan_keyword'attr' and scan_string() or nil
+  local page = scan_keyword'page' and scan_int() or nil
+  local userpassword = scan_keyword'userpassword' and scan_string() or nil
+  local ownerpassword = scan_keyword'ownerpassword' and scan_string() or nil
+  -- local colorspace = scan_keyword'colorspace' and scan_int() or nil -- Doesn't make sense for PDF
+  local pagebox = scan_keyword'mediabox' and 'media'
+               or scan_keyword'cropbox' and 'crop'
+               or scan_keyword'bleedbox' and 'bleed'
+               or scan_keyword'trimbox' and 'trim'
+               or scan_keyword'artbox' and 'art'
                or nil
-  local filename = token.scan_string()
+  local filename = scan_string()
   local img = imglib.scan{
     attr = attr,
     page = page,
@@ -856,14 +882,14 @@ token.luacmd("saveimageresource", function(imm)
   local pfile = get_pfile()
   lastimage = imglib.get_num(pfile, img)
   lastimagepages = img.pages or 1
-  if imm == 'immediate' then
+  if immediate == 8 then
     imglib_immediatewrite(pfile, img)
   end
-end, "protected")
+end, "value")
 
 token.luacmd("useimageresource", function()
   local pfile = get_pfile()
-  local img = assert(imglib.from_num(token.scan_int()))
+  local img = assert(imglib.from_num(scan_int()))
   imglib_write(pfile, img)
 end, "protected")
 
@@ -875,11 +901,11 @@ local integer_code = value_values.integer
 
 token.luacmd("lastsavedimageresourceindex", function()
   return integer_code, lastimage
-end, "protected", "value")
+end, "value")
 
 token.luacmd("lastsavedimageresourcepages", function()
   return integer_code, lastimagepages
-end, "protected", "value")
+end, "value")
 
 local savedbox = require'luametalatex-pdf-savedbox'
 local savedbox_save = savedbox.save
@@ -890,7 +916,7 @@ function tex.saveboxresource(n, attr, resources, immediate, type, margin, pfile)
       error[[Invalid argument to saveboxresource]]
     end
     token.put_next(token.create'box', token.new(n, token.command_id'char_given'))
-    n = token.scan_list()
+    n = scan_box()
   end
   margin = margin or pdfvariable.xformmargin
   return savedbox_save(pfile or get_pfile(), n, attr, resources, immediate, type, margin, fontdirs, usedglyphs)
@@ -899,41 +925,46 @@ tex.useboxresource = savedbox.use
 
 local lastbox = -1
 
-token.luacmd("saveboxresource", function(imm)
-  local type
-  if token.scan_keyword'type' then
-    texio.write_nl('XForm type attribute ignored')
-    type = token.scan_int()
+token.luacmd("saveboxresource", function(_, immediate)
+  if immediate == "value" then return end
+  if immediate and immediate & 0x7 ~= 0 then
+    immediate = immediate & 0x8
+    tex.error("Unexpected prefix", "You used \\saveboxresource with a prefix that doesn't belong there. I will ignore it for now.")
   end
-  local attr = token.scan_keyword'attr' and token.scan_string() or nil
-  local resources = token.scan_keyword'resources' and token.scan_string() or nil
-  local margin = token.scan_keyword'margin' and token.scan_dimen() or nil
-  local box = token.scan_int()
+  local type
+  if scan_keyword'type' then
+    texio.write_nl('XForm type attribute ignored')
+    type = scan_int()
+  end
+  local attr = scan_keyword'attr' and scan_string() or nil
+  local resources = scan_keyword'resources' and scan_string() or nil
+  local margin = scan_keyword'margin' and scan_dimen() or nil
+  local box = scan_int()
 
-  local index = tex.saveboxresource(box, attr, resources, imm == 'immediate', type, margin)
+  local index = tex.saveboxresource(box, attr, resources, immediate == 8, type, margin)
   lastbox = index
-end, "protected")
+end, "value")
 
 token.luacmd("useboxresource", function()
   local width, height, depth
   while true do
-    if token.scan_keyword'width' then
-      width = token.scan_dimen()
-    elseif token.scan_keyword'height' then
-      height = token.scan_dimen()
-    elseif token.scan_keyword'depth' then
-      depth = token.scan_dimen()
+    if scan_keyword'width' then
+      width = scan_dimen()
+    elseif scan_keyword'height' then
+      height = scan_dimen()
+    elseif scan_keyword'depth' then
+      depth = scan_dimen()
     else
       break
     end
   end
-  local index = token.scan_int()
+  local index = scan_int()
   node.write((tex.useboxresource(index, width, height, depth)))
 end, "protected")
 
 token.luacmd("lastsavedboxresourceindex", function()
   return integer_code, lastbox
-end, "protected", "value")
+end, "value")
 
 local saved_pos_x, saved_pos_y = -1, -1
 local save_pos_whatsit = declare_whatsit('save_pos', function(_, _, _, x, y)
@@ -945,11 +976,11 @@ end, "protected")
 
 token.luacmd("lastxpos", function()
   return integer_code, (saved_pos_x+.5)//1
-end, "protected", "value")
+end, "value")
 
 token.luacmd("lastypos", function()
   return integer_code, (saved_pos_y+.5)//1
-end, "protected", "value")
+end, "value")
 
 local function pdf_register_funcs(name)
   pdf[name] = ""
diff --git a/luametalatex-basecallbacks.lua b/luametalatex-basecallbacks.lua
index 9a2142f..89c6e6d 100644
--- a/luametalatex-basecallbacks.lua
+++ b/luametalatex-basecallbacks.lua
@@ -1,19 +1,19 @@
--- Two callbacks are defined in other files: pre_dump in lateinit and find_fmt_file in init
+-- Three callbacks are defined in other files: stop_run in back-pdf, pre_dump in lateinit, and find_fmt_file in init
 
 local read_tfm = font.read_tfm
 local font_define = font.define
-local callback_register = callback.register
+local callbacks = require'luametalatex-callbacks'
 
 if status.ini_version then
-  callback_register('define_font', function(name, size)
+  function callbacks.define_font(name, size)
     local f = read_tfm(name, size)
     if not f then return end
     local id = font_define(f)
     lua.prepared_code[#lua.prepared_code+1] = string.format("assert(%i == font.define(font.read_tfm(%q, %i)))", id, name, size)
     return id
-  end)
+  end
 else
-  callback_register('define_font', function(name, size)
+  function callbacks.define_font(name, size)
     local f = read_tfm(name, size)
     if not f then
       tex.error(string.format("Font %q not found", name), "The requested font could't be loaded.\n\z
@@ -22,40 +22,94 @@ else
       return 0
     end
     return font.define(f)
-  end)
+  end
+end
+callbacks.__freeze'define_font'
+
+function callbacks.find_log_file(name) return name end
+callbacks.__freeze'find_log_file'
+
+-- find_data_file is not an engine callback in luametatex, so we don't __freeze it
+if status.ini_version then
+  function unhook_expl()
+    callbacks.find_data_file = nil
+  end
+  function callbacks.find_data_file(name)
+    if name == 'ltexpl.ltx' then
+      name = 'luametalatex-ltexpl-hook'
+    end
+    return kpse.find_file(name, 'tex', true)
+  end
 end
-callback_register('find_log_file', function(name) return name end)
-do
   local function normal_find_data_file(name)
     return kpse.find_file(name, 'tex', true)
   end
-  if status.ini_version then
-    function unhook_expl()
-      callback_register('find_data_file', normal_find_data_file)
-    end
-    callback_register('find_data_file', function(name)
-      if name == 'ltexpl.ltx' then
-        name = 'luametalatex-ltexpl-hook'
-      end
-      return normal_find_data_file(name)
-    end)
+function callbacks.open_data_file(name)
+  local find_callback = callbacks.find_data_file
+  local path
+  if find_callback then
+    path = find_callback(name)
   else
-    callback_register('find_data_file', normal_find_data_file)
+    path = kpse.find_file(name, 'tex', true)
   end
-end
-callback_register('open_data_file', function(name)
-  local f = io.open(name, 'r')
-  return setmetatable({
+  if not path then return end
+
+  local open_callback = callbacks.open_data_file
+  if open_callback then
+    return open_callback(path)
+  end
+
+  local f = io.open(path, 'r')
+  return f and setmetatable({
     reader = function()
       local line = f:read()
       return line
     end,
-    close = function()error[[1]] return f:close() end,
+    close = function() f:close() f = nil end,
   }, {
-    __gc = function()f:close()end,
+    __gc = function() if f then f:close() end end,
   })
-end)
-callback_register('handle_error_hook', function()
+end
+callbacks.__freeze('open_data_file', true)
+
+local do_terminal_input do
+  local function terminal_open_data_file()
+    local old = callbacks.open_data_file
+    return function()
+      callbacks.open_data_file = old
+      return {
+        reader = function()
+          texio.write_nl('term', '* ')
+          local line = io.stdin:read()
+          return line
+        end,
+        close = function() end,
+      }
+    end
+  end
+  function do_terminal_input()
+    local old_find = callbacks.find_data_file
+    function callbacks.find_data_file(name) 
+      callbacks.find_data_file = old_find 
+      return name
+    end
+    callbacks.open_data_file = terminal_open_data_file()
+    token.put_next(token.create'expandafter', token.create'relax', token.create'input', 'TERMINAL ')
+    token.skip_next_expanded()
+  end
+end
+
+function callbacks.intercept_tex_error(mode, eof)
+  -- if eof then
+  --   print'EOF'
+  --   tex.runtoks(function()token.put_next(token.create'tracingall')end)
+  --   do_terminal_input()
+  --   tex.runtoks(token.skip_next)
+  --   return 3
+  -- end
+  texio.write'.'
+  tex.show_context()
+  if mode ~= 3 then return mode end
   repeat
     texio.write_nl'? '
     local line = io.read()
@@ -85,4 +139,5 @@ callback_register('handle_error_hook', function()
     end
   until false
   return 3
-end)
+end
+callbacks.__freeze'intercept_tex_error'
diff --git a/luametalatex-baseregisters.lua b/luametalatex-baseregisters.lua
index 384232d..fe0c165 100644
--- a/luametalatex-baseregisters.lua
+++ b/luametalatex-baseregisters.lua
@@ -1,5 +1,9 @@
+local scan_dimen = token.scan_dimen
+local scan_int = token.scan_integer
+local scan_keyword = token.scan_keyword
+
 local value_values = token.values'value'
-for i=0,#value_values do
+for i=0, #value_values do
   value_values[value_values[i]] = i
 end
 local count_code = value_values.integer
@@ -30,10 +34,10 @@ local function tex_variable(value, scanner, name, default)
     if scanning == 'value' then
       return value, tex_variables[name]
     else
-      token.scan_keyword'='
-      return set_local(tex_variables, name, scanner(), scanning == 'global')
+      scan_keyword'='
+      return set_local(tex_variables, name, scanner(), scanning and scanning & 4 == 4)
     end
-  end, 'global', 'protected', 'value')
+  end, 'global', 'value')
   if status.ini_version then
     tex_variables[name] = default
   end
@@ -99,14 +103,14 @@ local function pdf_variable(value, scanner, name, default, force_default)
     if scanning == 'value' then
       return value, real_pdf_variables[name]
     elseif force_default then
-      token.scan_keyword'='
+      scan_keyword'='
       local new = scanner()
       if new ~= default then
         texio.write_nl('term and log', string.format("Unsupported PDF variable: \z
             %q is not supported and fixed to %i, but you tried to set it to %i", name, default, new))
       end
     else
-      token.scan_keyword'='
+      scan_keyword'='
       return set_local(real_pdf_variables, name, scanner(), scanning == 'global')
     end
   end, 'global', 'protected', 'value')
@@ -115,43 +119,43 @@ local function pdf_variable(value, scanner, name, default, force_default)
   end
 end
 
-tex_variable(count_code, token.scan_int, 'suppressfontnotfounderror', 0)
-tex_variable(count_code, token.scan_int, 'outputmode', 1) -- The "traditional" default would be 0,
+tex_variable(count_code, scan_int, 'suppressfontnotfounderror', 0)
+tex_variable(count_code, scan_int, 'outputmode', 1) -- The "traditional" default would be 0,
                                                           -- but we do not actually support that.
-tex_variable(dimen_code, token.scan_dimen, 'pageheight', 0)
-tex_variable(dimen_code, token.scan_dimen, 'pagewidth', 0)
+tex_variable(dimen_code, scan_dimen, 'pageheight', 0)
+tex_variable(dimen_code, scan_dimen, 'pagewidth', 0)
 
-tex_variable(count_code, token.scan_int, 'bodydirection', 0)
-tex_variable(count_code, token.scan_int, 'pagedirection', 0)
+tex_variable(count_code, scan_int, 'bodydirection', 0)
+tex_variable(count_code, scan_int, 'pagedirection', 0)
 
-pdf_variable(dimen_code, token.scan_dimen, 'horigin', tex.sp'1in')
-pdf_variable(dimen_code, token.scan_dimen, 'vorigin', tex.sp'1in')
-pdf_variable(dimen_code, token.scan_dimen, 'linkmargin', tex.sp'0pt')
-pdf_variable(dimen_code, token.scan_dimen, 'destmargin', tex.sp'0pt')
-pdf_variable(dimen_code, token.scan_dimen, 'xformmargin', tex.sp'0pt')
-pdf_variable(dimen_code, token.scan_dimen, 'threadmargin', tex.sp'0pt', true) -- We don't support threads, so this isn't doing anything
-pdf_variable(count_code, token.scan_int, 'majorversion', 1)
-pdf_variable(count_code, token.scan_int, 'minorversion', 7)
-pdf_variable(count_code, token.scan_int, 'compresslevel', 9)
-pdf_variable(count_code, token.scan_int, 'objcompresslevel', 3)
+pdf_variable(dimen_code, scan_dimen, 'horigin', tex.sp'1in')
+pdf_variable(dimen_code, scan_dimen, 'vorigin', tex.sp'1in')
+pdf_variable(dimen_code, scan_dimen, 'linkmargin', tex.sp'0pt')
+pdf_variable(dimen_code, scan_dimen, 'destmargin', tex.sp'0pt')
+pdf_variable(dimen_code, scan_dimen, 'xformmargin', tex.sp'0pt')
+pdf_variable(dimen_code, scan_dimen, 'threadmargin', tex.sp'0pt', true) -- We don't support threads, so this isn't doing anything
+pdf_variable(count_code, scan_int, 'majorversion', 1)
+pdf_variable(count_code, scan_int, 'minorversion', 7)
+pdf_variable(count_code, scan_int, 'compresslevel', 9)
+pdf_variable(count_code, scan_int, 'objcompresslevel', 3)
 
-pdf_variable(count_code, token.scan_int, 'decimaldigits', 4, true) -- Will probably stay fixed, but should be more consistent
-pdf_variable(count_code, token.scan_int, 'gentounicode', 0, true) -- We expect the fontloader to generade tounicode tables. Might change at some point
+pdf_variable(count_code, scan_int, 'decimaldigits', 4, true) -- Will probably stay fixed, but should be more consistent
+pdf_variable(count_code, scan_int, 'gentounicode', 0, true) -- We expect the fontloader to generade tounicode tables. Might change at some point
 -- These two are ignored, but that is consistent with pdfTeX as long as imageapplygamma is 0:
-pdf_variable(count_code, token.scan_int, 'gamma', 1000)
-pdf_variable(count_code, token.scan_int, 'imagegamma', 1000)
-pdf_variable(count_code, token.scan_int, 'imageapplygamma', 0, true)
-pdf_variable(count_code, token.scan_int, 'imagehicolor', 1, true) -- We don't consider ancient PDF versions, no no reason to strip images
-pdf_variable(count_code, token.scan_int, 'imageaddfilename', 0, true) -- Could be added, but I never saw a reason for this anyway.
-pdf_variable(count_code, token.scan_int, 'inclusionerrorlevel', -1, true) -- FIXME: At least a warning should be supported
-pdf_variable(count_code, token.scan_int, 'inclusioncopyfonts', 0, true) -- Would be fragile and restrict our ability to use "creative" font constructs
-pdf_variable(count_code, token.scan_int, 'uniqueresname', 0, true) -- I add this if you show me a usecase
-pdf_variable(count_code, token.scan_int, 'pagebox', 2, true) -- TODO (1: media, 2: crop, 3: bleed, 4: trim, 5: art
-pdf_variable(count_code, token.scan_int, 'forcepagebox', 0, true) -- Considered obsolete even in pdfTeX
-pdf_variable(count_code, token.scan_int, 'imageresolution', 72, true) -- TODO Also 0 should be the same as 72 ?!?!?!?
+pdf_variable(count_code, scan_int, 'gamma', 1000)
+pdf_variable(count_code, scan_int, 'imagegamma', 1000)
+pdf_variable(count_code, scan_int, 'imageapplygamma', 0, true)
+pdf_variable(count_code, scan_int, 'imagehicolor', 1, true) -- We don't consider ancient PDF versions, no no reason to strip images
+pdf_variable(count_code, scan_int, 'imageaddfilename', 0, true) -- Could be added, but I never saw a reason for this anyway.
+pdf_variable(count_code, scan_int, 'inclusionerrorlevel', -1, true) -- FIXME: At least a warning should be supported
+pdf_variable(count_code, scan_int, 'inclusioncopyfonts', 0, true) -- Would be fragile and restrict our ability to use "creative" font constructs
+pdf_variable(count_code, scan_int, 'uniqueresname', 0, true) -- I add this if you show me a usecase
+pdf_variable(count_code, scan_int, 'pagebox', 2, true) -- TODO (1: media, 2: crop, 3: bleed, 4: trim, 5: art
+pdf_variable(count_code, scan_int, 'forcepagebox', 0, true) -- Considered obsolete even in pdfTeX
+pdf_variable(count_code, scan_int, 'imageresolution', 72, true) -- TODO Also 0 should be the same as 72 ?!?!?!?
 
-pdf_variable(count_code, token.scan_int, 'pkresolution', 1200) -- Original default is 72, but that's crazy
-pdf_variable(count_code, token.scan_int, 'pkfixeddpi', 0) -- TODO: Implemented, but even when set to one, font sharing doesn't adapt yet.
+pdf_variable(count_code, scan_int, 'pkresolution', 1200) -- Original default is 72, but that's crazy
+pdf_variable(count_code, scan_int, 'pkfixeddpi', 0) -- TODO: Implemented, but even when set to one, font sharing doesn't adapt yet.
                                                           -- Changing that is complicated because it has to be known pretty early.
 pdf_toks('pkmode', '')
 
diff --git a/luametalatex-build-bytecode.lua b/luametalatex-build-bytecode.lua
index ac05de4..cd53bef 100644
--- a/luametalatex-build-bytecode.lua
+++ b/luametalatex-build-bytecode.lua
@@ -14,7 +14,7 @@ first = first .. later
 
 local list = {}
 
-return function(t)
+return true and function(t) return '' end or function(t)
   local length = #t
   local tmpl = first
   for i, mod in ipairs(t) do
diff --git a/luametalatex-callbacks.lua b/luametalatex-callbacks.lua
index 509f6db..ed60d21 100644
--- a/luametalatex-callbacks.lua
+++ b/luametalatex-callbacks.lua
@@ -1,24 +1,69 @@
 -- Now overwrite the callback functionality. Our system is based on the ssumption there there are
 -- no unknown callback names, just callbacks very unlikely to ever be called. That doesn't lead to
 -- good error checking, but we expect this to be overwritten by LaTeX anyway.
+--
+-- There are four callback types on this level:
+--  1. luametalatex defined callbacks. They are not real engine callbacks, the code using them is
+--     responsible for their potential non-existance.
+--  2. Engine callbacks not defined by us. They are simply passed on to the engine. All engine
+--     callbacks are set to this by default.
+--  3. Engine callbacks with a provided default. There is a luametalatex implementation, but it
+--     can be overwritten by the user. If the user disabled their implementation, so provided
+--     default is restored.
+--  4. Engine callbacks with mandatory code. The luametalatex implementation can not be overwitten
+--     by the user, but a luametalatex-defined callback is added with the same name.
+--
+--  A callback has type 1 or type 4 if is_user_callback is true. If it has type 4, is_user_callback
+--  has to be set manually and in addition, an implementation if the system callback is registered.
+--
+--  A callback has type 3, if is_user_callback is false and system_callbacks is defined.
 
+local callback_known = callback.known
 local callback_find = callback.find
 local callback_register = callback.register
 local rawset = rawset
-local callbacks = setmetatable({}, {
+local system_callbacks = {}
+local is_user_callback = setmetatable({}, {
+  __index = function(t, name)
+    local is_user = not callback_known(name)
+    t[name] = is_user
+    return is_user
+  end,
+})
+local callbacks = setmetatable({
+  __freeze = function(name, fixed)
+    -- Convert from type 2 to type 3 or 4. This function will be deleted before user code runs.
+    assert(not is_user_callback[name], 'Not a system callback')
+    assert(not system_callbacks[name], 'Already frozen')
+    is_user_callback[name] = fixed and true or false
+    system_callbacks[name] = callback_find(name)
+    assert(system_callbacks[name], 'Attempt to freeze undefined callback')
+  end,
+}, {
   __index = function(cbs, name)
+    if is_user_callback[name] then
+      -- Avoid repetitive lookups
+      rawset(cbs, name, false)
+      return false
+    end
     return callback_find(name)
   end,
   __newindex = function(cbs, name, new)
-    return callback_register(name, new) or rawset(cbs, name, new)
+    if is_user_callback[name] then
+      -- Avoid repetitive lookups
+      rawset(cbs, name, new or false)
+      return
+    end
+    return callback_register(name, new or system_callbacks[name])
   end,
 })
 
 function callback.register(name, new)
   callbacks[name] = new
 end
-function callback.find(name)
-  return callbacks[name]
+-- The and ... or construction makes sure that even in raw mode, non-engine callbacks are found
+function callback.find(name, raw)
+  return raw and callback_find(name) or callbacks[name]
 end
 
 return callbacks
diff --git a/luametalatex-dir-registers.lua b/luametalatex-dir-registers.lua
index 2c4abcf..abc3804 100644
--- a/luametalatex-dir-registers.lua
+++ b/luametalatex-dir-registers.lua
@@ -1,3 +1,6 @@
+local scan_int = token.scan_integer
+local scan_keyword = token.scan_keyword
+
 -- local names = {}
 local setters = {
 }
@@ -14,19 +17,18 @@ function tex.getpardir()  return tex.pardirection  end
 local integer_code = value_values.integer
 local function set_xdir(id, scanning)
   if scanning == 'value' then
-    print(scanning)
     return integer_code, getters[id]()
   end
   -- local global = scanning == 'global'
   local value
-  if token.scan_keyword'tlt' then
+  if scan_keyword'tlt' then
     value = 0
-  elseif token.scan_keyword'trt' then
+  elseif scan_keyword'trt' then
     value = 1
   else
-    value = token.scan_int()
+    value = scan_int()
   end
-  setters[id](value)
+  setters[id](value, scanning)
 end
 return function(name)
   local getter = tex["get" .. name]
diff --git a/luametalatex-firstcode.lua b/luametalatex-firstcode.lua
index 4e8fc55..e02e6b5 100644
--- a/luametalatex-firstcode.lua
+++ b/luametalatex-firstcode.lua
@@ -1,3 +1,12 @@
+local scan_int = token.scan_integer
+local scan_token = token.scan_token
+local scan_tokenlist = token.scan_tokenlist
+local scan_keyword = token.scan_keyword
+local scan_csname = token.scan_csname
+local set_macro = token.set_macro
+
+local callback_find = callback.find
+
 local lua_call_cmd = token.command_id'lua_call'
 local properties = node.direct.get_properties_table()
 node.direct.properties = properties
@@ -20,7 +29,7 @@ local function scan_filename()
   local quoted = false
   local tok, cmd
   repeat
-    tok = token.scan_token()
+    tok = scan_token()
     cmd = tok.command
   until cmd ~= spacer_cmd and cmd ~= relax_cmd
   while (tok.command <= 12 and tok.command > 0 and tok.command ~= 9 and tok.index <= token.biggest_char()
@@ -31,39 +40,68 @@ local function scan_filename()
     else
       name[#name+1] = tok.index
     end
-    tok = token.scan_token()
+    tok = scan_token()
   end
   return utf8.char(table.unpack(name))
 end
 
+-- These are chosen to coincide with ltluatex's default catcodetables.
+-- In expl-hook, we check that the values are as expected.
+local initex_catcodetable = 1
+local string_catcodetable = 2
+if status.ini_version then
+  tex.runtoks(function()tex.sprint[[\initcatcodetable 1\initcatcodetable 2]]end)
+  local setcatcode = tex.setcatcode
+  for i=0,127 do
+    setcatcode('global', 2, i, 12)
+  end
+  setcatcode('global', 2, 32, 10)
+end
+
 local l = lpeg or require'lpeg'
 local add_file_extension = l.Cs((1-('.' * (1-l.S'./\\')^0) * -1)^0 * (l.P(1)^1+l.Cc'.tex'))
-local ofiles = {}
+local ofiles, ifiles = {}, {}
 local function do_openout(p)
   if ofiles[p.file] then
-    error[[Existing file]]
-  else
-    local msg
-    ofiles[p.file], msg = io.open(add_file_extension:match(p.name), 'w')
-    if not ofiles[p.file] then
-      error(msg)
-    end
+    ofiles[p.file]:close()
+  end
+  local msg
+  ofiles[p.file], msg = io.open(add_file_extension:match(p.name), 'w')
+  if not ofiles[p.file] then
+    error(msg)
   end
 end
 local open_whatsit = new_whatsit('open', do_openout)
 token.luacmd("openout", function(_, immediate) -- \openout
-  local file = token.scan_int()
-  token.scan_keyword'='
+  if immediate == "value" then return end
+  if immediate and immediate & 0x7 ~= 0 then
+    immediate = immediate & 0x8
+    tex.error("Unexpected prefix", "You used \\openout with a prefix that doesn't belong there. I will ignore it for now.")
+  end
+  local file = scan_int()
+  scan_keyword'='
   local name = scan_filename()
   local props = {file = file, name = name}
-  if immediate == "immediate" then
+  if immediate and immediate == 8 then
     do_openout(props)
   else
     local whatsit = node.direct.new(whatsit_id, open_whatsit)
     properties[whatsit] = props
     node.direct.write(whatsit)
   end
-end, "protected")
+end, "value")
+token.luacmd("openin", function(_, prefix)
+  if prefix == "value" then return end
+  local file = scan_int()
+  scan_keyword'='
+  local name = scan_filename()
+  if ifiles[file] then
+    ifiles[file]:close()
+  end
+  local msg
+  ifiles[file] = callback_find('open_data_file', true)(name) -- raw to pick up our wrapper which handles defaults and finding the file
+end, "value")
+
 local function do_closeout(p)
   if ofiles[p.file] then
     ofiles[p.file]:close()
@@ -72,16 +110,30 @@ local function do_closeout(p)
 end
 local close_whatsit = new_whatsit('close', do_closeout)
 token.luacmd("closeout", function(_, immediate) -- \closeout
-  local file = token.scan_int()
+  if immediate == "value" then return end
+  if immediate and immediate & 0x7 ~= 0 then
+    immediate = immediate & 0x8
+    tex.error("Unexpected prefix", "You used \\closeout with a prefix that doesn't belong there. I will ignore it for now.")
+  end
+  local file = scan_int()
   local props = {file = file}
-  if immediate == "immediate" then
+  if immediate == 8 then
     do_closeout(props)
   else
     local whatsit = node.direct.new(whatsit_id, close_whatsit)
     properties[whatsit] = props
     node.direct.write(whatsit)
   end
-end, "protected")
+end, "value")
+token.luacmd("closein", function(_, prefix)
+  if prefix == "value" then return end
+  local file = scan_int()
+  if ifiles[file] then
+    ifiles[file]:close()
+    ifiles[file] = nil
+  end
+end, "value")
+
 local function do_write(p)
   local content = token.to_string(p.data) .. '\n'
   local file = ofiles[p.file]
@@ -93,17 +145,127 @@ local function do_write(p)
 end
 local write_whatsit = new_whatsit('write', do_write)
 token.luacmd("write", function(_, immediate) -- \write
-  local file = token.scan_int()
-  local content = token.scan_tokenlist()
+  if immediate == "value" then return end
+  if immediate and immediate & 0x7 ~= 0 then
+    immediate = immediate & 0x8
+    tex.error("Unexpected prefix", "You used \\write with a prefix that doesn't belong there. I will ignore it for now.")
+  end
+  local file = scan_int()
+  local content = scan_tokenlist()
   local props = {file = file, data = content}
-  if immediate == "immediate" then
+  if immediate == 8 then
     do_write(props)
   else
     local whatsit = node.direct.new(whatsit_id, write_whatsit)
     properties[whatsit] = props
     node.direct.write(whatsit)
   end
-end, "protected")
+end, "value")
+
+local undefined_tok = token.new(0, token.command_id'undefined_cs')
+local prefix_cmd = token.command_id'prefix'
+local function prefix_to_tokens(prefix)
+  if not prefix then return end
+  for i=2, 0, -1 do
+    if prefix & (1<<i) ~= 0 then
+      token.put_next(token.new(i, prefix_cmd))
+    end
+  end
+end
+local expand_after = token.primitive_tokens.expandafter
+local input_tok = token.primitive_tokens.input
+local endlocalcontrol = token.primitive_tokens.endlocalcontrol
+local afterassignment = token.primitive_tokens.afterassignment
+local lbrace = token.new(0, 1)
+local rbrace = token.new(0, 2)
+token.luacmd("read", function(_, prefix)
+  if immediate == "value" then return end
+  local id = scan_int()
+  if not scan_keyword'to' then
+    tex.error("Missing `to' inserted", "You should have said `\\read<number> to \\cs'.\nI'm going to look for the \\cs now.")
+  end
+  local macro = scan_csname(true)
+  local file = ifiles[id]
+  local line
+  if file then
+    line = file:reader()
+    if not line then
+      file:close()
+      ifiles[id] = nil
+    end
+  else
+    error[[FIXME: Ask the user for input]]
+  end
+  local endlocal
+  tex.runtoks(function()
+    endlocal = token.scan_next()
+    tex.sprint(endlocal)
+    tex.print(line and line ~= "" and line or " ")
+    tex.print(endlocal)
+  end)
+  local tokens = {}
+  local balance = 0
+  while true do
+    local tok = token.scan_next()
+    if tok == endlocal then break end
+    if tok.command == 1 then
+      balance = balance + 1
+    elseif tok.command == 2 then
+      balance = balance - 1
+    end
+    tokens[#tokens+1] = tok
+  end
+  if balance ~= 0 then error[[FIXME: Read additional input lines]] end
+  tex.runtoks(function()
+    tokens[#tokens+1] = rbrace
+    token.put_next(tokens)
+    token.put_next(token.primitive_tokens.def, token.create(macro), lbrace)
+    prefix_to_tokens(prefix)
+  end)
+end, "value")
+
+token.luacmd("readline", function(_, prefix)
+  if immediate == "value" then return end
+  local id = scan_int()
+  if not scan_keyword'to' then
+    tex.error("Missing `to' inserted", "You should have said `\\read<number> to \\cs'.\nI'm going to look for the \\cs now.")
+  end
+  local macro = scan_csname(true)
+  local file = ifiles[id]
+  local line
+  if file then
+    line = file:reader()
+    if not line then
+      file:close()
+      ifiles[id] = nil
+    end
+  else
+    error[[FIXME: Ask the user for input]]
+  end
+  line =  line and line:match"^(.*[^ ])[ ]*$"
+  local endlinechar = tex.endlinechar
+  if endlinechar >= 0 and endlinechar < 0x80 then
+    line = (line or '') .. string.char(endlinechar)
+  end
+  set_macro(string_catcodetable, macro, line or '', prefix)
+end, "value")
+
+local integer_code, boolean_code do
+  local value_values = token.values'value'
+  for i=0,#value_values do
+    if value_values[i] == "integer" then
+      integer_code = i
+    elseif value_values[i] == "boolean" then
+      boolean_code = i
+    end
+  end
+end
+
+token.luacmd("ifeof", function(_)
+  local id = scan_int()
+  return boolean_code, not ifiles[id]
+end, "condition")
+
 local late_lua_whatsit = new_whatsit('late_lua', function(p, pfile, n, x, y)
   local code = p.data
   if not code then
@@ -116,8 +278,8 @@ local late_lua_whatsit = new_whatsit('late_lua', function(p, pfile, n, x, y)
   end
   return pdf._latelua(pfile, x, y, code)
 end)
-token.luacmd("latelua", function(_) -- \latelua
-  local content = token.scan_tokenlist()
+token.luacmd("latelua", function() -- \latelua
+  local content = scan_tokenlist()
   local props = {token = content}
   local whatsit = node.direct.new(whatsit_id, late_lua_whatsit)
   properties[whatsit] = props
@@ -125,35 +287,21 @@ token.luacmd("latelua", function(_) -- \latelua
 end, "protected")
 
 local functions = lua.get_functions_table()
-token.luacmd("immediate", function() -- \immediate
-  local next_tok = token.scan_token()
-  if next_tok.command ~= lua_call_cmd then
-    return token.put_next(next_tok)
-  end
-  local function_id = next_tok.index
-  return functions[function_id](function_id, 'immediate')
-end, "protected")
 
 require'luametalatex-baseregisters'
 require'luametalatex-back-pdf'
 require'luametalatex-node-luaotfload'
 
-local integer_code do
-  local value_values = token.values'value'
-  for i=0,#value_values do
-    if value_values[i] == "integer" then
-      integer_code = i
-      break
-    end
-  end
-end
 token.luacmd("Umathcodenum", function(_, scanning)
   if scanning then
-    local class, family, char = tex.getmathcodes (token.scan_int())
+    local class, family, char = tex.getmathcodes (scan_int())
     return integer_code, char | (class | family << 3) << 21
   else
-    local char = token.scan_int()
-    local mathcode = token.scan_int()
+    local char = scan_int()
+    local mathcode = scan_int()
     tex.setmathcodes(char, (mathcode >> 21) & 7, mathcode >> 24, mathcode & 0x1FFFFF)
   end
 end, "force", "global", "value")
+
+-- This is effectivly the last line before we hand over to normal TeX.
+require'luametalatex-callbacks'.__freeze = nil
diff --git a/luametalatex-font-resolve.lua b/luametalatex-font-resolve.lua
index eb973b3..4540e5e 100644
--- a/luametalatex-font-resolve.lua
+++ b/luametalatex-font-resolve.lua
@@ -2,7 +2,7 @@ font.read_tfm = require'luametalatex-font-tfm'
 local read_vf = require'luametalatex-font-vf'
 font.read_vf = read_vf
 local fontmap = require'luametalatex-pdf-font-map'.fontmap
-local callback_find = callback.find
+local callbacks = require'luametalatex-callbacks'
 
 local old_font_define = font.define
 local old_addcharacters = font.addcharacters
@@ -78,7 +78,7 @@ function font.define(f)
   if f.fonts then
     for i, f in next, f.fonts do
       if not f.id then
-        f.id = assert(callback_find'define_font'(f.name, f.size or -1000))
+        f.id = assert(callbacks.define_font(f.name, f.size or -1000))
       elseif f.id == 0 then
         f.id = id
       end
@@ -96,7 +96,7 @@ function font.addcharacters(fid, newdir)
       fonts_map = {}
       for i,f in next, newdir.fonts do
         if not f.id then
-          f.id = assert(callback_find'define_font'(f.name, f.size or -1000))
+          f.id = assert(callback.define_font(f.name, f.size or -1000))
         elseif f.id == 0 then
           f.id = fid
         end
diff --git a/luametalatex-lateinit.lua b/luametalatex-lateinit.lua
index a60ada2..c83939a 100644
--- a/luametalatex-lateinit.lua
+++ b/luametalatex-lateinit.lua
@@ -14,6 +14,7 @@ pdf = {
 }
 require'luametalatex-font-resolve' -- Replace font.define. Must be loaded before callbacks
 require'luametalatex-basecallbacks'
+local callbacks = require'luametalatex-callbacks'
 
 local primitives = {}
 do
@@ -54,6 +55,7 @@ local undefined_cmd = token.command_id'undefined_cs'
 local lua_call_cmd = token.command_id'lua_call'
 local lua_value_cmd = token.command_id'lua_value'
 local lua_expandable_call_cmd = token.command_id'lua_expandable_call'
+local if_test_cmd = token.command_id'if_test'
 function token.luacmd(name, func, ...)
   local idx
   local tok = token.create(name)
@@ -64,6 +66,8 @@ function token.luacmd(name, func, ...)
     idx = tok.index
   elseif cmd == lua_expandable_call_cmd then
     idx = tok.index
+  elseif cmd == if_test_cmd and tok.index > 48 then
+    idx = tok.index - 48
   elseif ... == 'force' then
     idx = new_luafunction(name)
     set_lua(name, idx, select(2, ...))
@@ -82,7 +86,10 @@ end
 
 if initex then
   local build_bytecode = nil -- To be filled
-  callback.register('pre_dump', function()
+  function callbacks.pre_dump()
+    local user_callback = callbacks.pre_dump
+    if user_callback then user_callback() end
+
     local prepared = lua.prepared_code
     prepared[1] = string.format("fixupluafunctions(%i)", predefined_luafunctions)
     for i=0,0 do -- maybeFIXME: In practise only one language is preloaded in LuaTeX anyway
@@ -121,7 +128,8 @@ if initex then
       end
     end
     lua.bytecode[tex.count[262]+1] = build_bytecode(table.concat(prepared, '\n'))
-  end)
+  end
+  callbacks.__freeze('pre_dump', true)
   return function(f)
     build_bytecode = f
     return require'luametalatex-firstcode'
diff --git a/luametalatex-ltexpl-hook.tex b/luametalatex-ltexpl-hook.tex
index 807a39e..74d45ba 100644
--- a/luametalatex-ltexpl-hook.tex
+++ b/luametalatex-ltexpl-hook.tex
@@ -1,6 +1,10 @@
 \directlua{unhook_expl()}
 % See baseregisters for list of toks pdfvariables
 \directlua{initialize_pdf_toks()}
+% We hardcode the ids of two catcodetables. If they ever change in the format, abort.
+\if 0\ifnum1=\catcodetable@initex\else\expandafter1\fi\ifnum2=\catcodetable@string\else\expandafter1\fi 0\else
+  \errmessage{Inconsistent catcodetable identifiers}
+\fi
 \ifx\@tfor\undefined
   \def\@tfor#1\do#2{}
 \fi
diff --git a/luametalatex-microtype.sty b/luametalatex-microtype.sty
index b7b71f9..456c285 100644
--- a/luametalatex-microtype.sty
+++ b/luametalatex-microtype.sty
@@ -24,10 +24,10 @@
   \noexpand\luametalatex@@everyjobandnow{\noexpand\directlua{!
     lua.get_functions_table()[\the\luametalatex@@expandglyphsinfont] = function()
       token.put_next(token.create'fontid')
-      local font = token.scan_int()
-      local stretch = token.scan_int()
-      local shrink = token.scan_int()
-      local step = token.scan_int()
+      local font = token.scan_integer()
+      local stretch = token.scan_integer()
+      local shrink = token.scan_integer()
+      local step = token.scan_integer()
       token.set_macro('pickup@font@@hook@luametalatex@microtype@' .. font, string.format("{}{%i}{%i}{%i}", stretch, shrink, step), "global")
     end
   }}
diff --git a/luametalatex-pdf.lua b/luametalatex-pdf.lua
index 352bd70..0e85f2e 100644
--- a/luametalatex-pdf.lua
+++ b/luametalatex-pdf.lua
@@ -10,6 +10,7 @@ local assigned = {}
 local delayed = {}
 local compress = xzip.compress
 local pdfvariable = pdf.variable
+local digest = sha2.digest256
 -- slightly tricky interface: No/nil return means that the objects content
 -- isn't known yet, while false indicates a delayed object.
 local function written(pdf, num)