Implement \tracingoutput

luametalatex-back-pdf.lua

@@ -1,4 +1,6 @@
-local scan_int = token.scan_integer
+local lmlt = luametalatex
+
+local scan_int = token.scan_int
 local scan_token = token.scan_token
 local scan_keyword = token.scan_keyword
 local scan_string = token.scan_string
@@ -22,6 +24,10 @@ local utils = require'luametalatex-pdf-utils'
 local strip_floats = utils.strip_floats
 local to_bp = utils.to_bp
 
+local immediate_flag = lmlt.flag.immediate
+
+local initex_catcodetable = 1
+
 local pdfname, pfile
 local fontdirs = setmetatable({}, {__index=function(t, k)t[k] = pfile:getobj() return t[k] end})
 local nodefont_meta = {}
@@ -47,10 +53,15 @@ local colorstacks = {{
     page_stack = {"0 g 0 G"},
   }}
 local spacer_cmd = token.command_id'spacer'
+local output_directory = arg['output-directory']
+local dir_sep = '/' -- FIXME
 local function get_pfile()
   if not pfile then
     pdfname = tex.jobname .. '.pdf'
-    pfile = newpdf.open(tex.jobname .. '.pdf')
+    if output_directory then
+      pdfname = output_directory .. dir_sep .. pdfname
+    end
+    pfile = newpdf.open(pdfname)
   end
   return pfile
 end
@@ -63,24 +74,31 @@ local function get_outline()
   return outline
 end
 local properties = node.direct.properties
-local immediateassignment = token.new(5, token.command_id'convert')
+local begin_group do
-local global = token.new(0, token.command_id'prefix')
+  local tokens = lmlt.primitive_tokens.begingroup
-local deadcycles = token.new(8, token.command_id'set_page_property')
+  function begin_group()
-local zero_tok = token.create(0x30)
+    token.put_next(tokens)
-local relax = token.new(0, 0)
+  end
-local reset_deadcycles = {
+end
-  immediateassignment,
+local finalize_shipout do
-  global,
+  local showbox_token = lmlt.primitive_tokens.showbox
-  deadcycles,
+  local global_token = lmlt.primitive_tokens.global
-  zero_tok,
+  local deadcycles_token = lmlt.primitive_tokens.deadcycles
-  relax,
+  local endgroup_token = lmlt.primitive_tokens.endgroup
-}
+  function finalize_shipout()
-token.luacmd("shipout", function()
+    if tex.tracingoutput > 0 then
+      tex.sprint(initex_catcodetable, showbox_token, 'content diagnose 0')
+    end
+    tex.sprint(initex_catcodetable, global_token, deadcycles_token, '0', endgroup_token)
+  end
+end
+lmlt.luacmd("shipout", function()
+  local outlist = scan_box()
   local pfile = get_pfile()
   local total_voffset, total_hoffset = tex.voffset + pdfvariable.vorigin, tex.hoffset + pdfvariable.horigin
   local voff = node.new'kern'
   voff.kern = total_voffset
-  voff.next = scan_box()
+  voff.next = outlist
   voff.next.shift = total_hoffset
   local list = node.direct.tonode(node.direct.vpack(node.direct.todirect(voff)))
   local pageheight, pagewidth = tex.pageheight, tex.pagewidth
@@ -94,9 +112,11 @@ token.luacmd("shipout", function()
   cur_page = nil
   local content = pfile:stream(nil, '', out)
   pfile:indirect(page, string.format([[<</Type/Page/Parent %i 0 R/Contents %i 0 R/MediaBox[0 %i %i %i]/Resources%s%s%s%s>>]], parent, content, -math.ceil(to_bp(list.depth)), math.ceil(to_bp(list.width)), math.ceil(to_bp(list.height)), resources(pdfvariable.pageresources .. pdf.pageresources), annots, pdfvariable.pageattr, pdf.pageattributes))
-  node.flush_list(list)
+  tex.runlocal(begin_group)
-  token.put_next(reset_deadcycles)
+  tex.box[0] = outlist
-  scan_token()
+  list.head.next = nil
+  node.free(list)
+  tex.runlocal(finalize_shipout)
 end, 'force', 'protected')
 
 local infodir = ""
@@ -226,7 +246,8 @@ function callbacks.stop_run()
   local size = pfile:close()
   texio.write_nl("term", "(see the transcript file for additional information)")
   -- TODO: Additional logging, epecially targeting the log file
-  texio.write_nl("term and log", string.format(" %d words of node memory still in use:", status.nodestate.use))
+  texio.write_nl("term and log", " node memory still in use:")
+  -- texio.write_nl("term and log", string.format(" %d words of node memory still in use:", status.nodestate.use))
   local by_type, by_sub = {}, {}
   for n, id, sub in node.traverse(node.usedlist()) do
     if id == whatsit_id then
@@ -249,7 +270,7 @@ function callbacks.stop_run()
 end
 callbacks.__freeze('stop_run', true)
 
-token.luacmd("pdfvariable", function()
+lmlt.luacmd("pdfvariable", function()
   for _, n in ipairs(pdf.variable_names) do
     if scan_keyword(n) then
       return token.put_next(token.create('pdfvariable  ' .. n))
@@ -639,7 +660,7 @@ local function maybe_gobble_cmd(cmd)
     token.put_next(t)
   end
 end
-token.luacmd("pdffeedback", function()
+lmlt.luacmd("pdffeedback", function()
   if scan_keyword"colorstackinit" then
     local page = scan_keyword'page'
               or (scan_keyword'nopage' and false) -- If you want to pass "page" as mode
@@ -658,10 +679,10 @@ token.luacmd("pdffeedback", function()
     error(string.format("Unknown PDF feedback %s", scan_word()))
   end
 end)
-token.luacmd("pdfextension", function(_, immediate)
+lmlt.luacmd("pdfextension", function(_, immediate)
   if immediate == "value" then return end
-  if immediate and immediate & 0x7 ~= 0 then
+  if immediate and immediate & ~immediate_flag ~= 0 then
-    immediate = immediate & 0x8
+    immediate = immediate & immediate_flag
     tex.error("Unexpected prefix", "You used \\pdfextension with a prefix that doesn't belong there. I will ignore it for now.")
   end
   if scan_keyword"colorstack" then
@@ -729,7 +750,7 @@ token.luacmd("pdfextension", function(_, immediate)
       local attr = scan_keyword'stream' and (scan_keyword'attr' and scan_string() or '')
       local isfile = scan_keyword'file'
       local content = scan_string()
-      if immediate == 8 then
+      if immediate == immediate_flag then
         if attr then
           pfile:stream(num, attr, content, isfile)
         else
@@ -828,6 +849,8 @@ token.luacmd("pdfextension", function(_, immediate)
       error[[Unsupported dest type]]
     end
     node.write(whatsit)
+  elseif scan_keyword'mapfile' then
+    fontmap.mapfile(scan_string())
   elseif scan_keyword'mapline' then
     fontmap.mapline(scan_string())
   else
@@ -852,11 +875,10 @@ local lastimage = -1
 local lastimagepages = -1
 
 -- These are very minimal right now but LaTeX isn't using the scaling etc. stuff anyway.
-token.luacmd("saveimageresource", function(_, immediate)
+lmlt.luacmd("saveimageresource", function(_, immediate)
   if immediate == "value" then return end
-  if immediate and immediate & 0x7 ~= 0 then
+  if immediate and immediate & ~immediate_flag ~= 0 then
-    print(immediate)
+    immediate = immediate & immediate_flag
-    immediate = immediate & 0x8
     tex.error("Unexpected prefix", "You used \\saveimageresource with a prefix that doesn't belong there. I will ignore it for now.")
   end
   local attr = scan_keyword'attr' and scan_string() or nil
@@ -882,24 +904,24 @@ token.luacmd("saveimageresource", function(_, immediate)
   local pfile = get_pfile()
   lastimage = imglib.get_num(pfile, img)
   lastimagepages = img.pages or 1
-  if immediate == 8 then
+  if immediate == immediate_flag then
     imglib_immediatewrite(pfile, img)
   end
 end, "value")
 
-token.luacmd("useimageresource", function()
+lmlt.luacmd("useimageresource", function()
   local pfile = get_pfile()
   local img = assert(imglib.from_num(scan_int()))
   imglib_write(pfile, img)
 end, "protected")
 
-local integer_code = token.value.integer
+local integer_code = lmlt.value.integer
 
-token.luacmd("lastsavedimageresourceindex", function()
+lmlt.luacmd("lastsavedimageresourceindex", function()
   return integer_code, lastimage
 end, "value")
 
-token.luacmd("lastsavedimageresourcepages", function()
+lmlt.luacmd("lastsavedimageresourcepages", function()
   return integer_code, lastimagepages
 end, "value")
 
@@ -921,10 +943,10 @@ tex.useboxresource = savedbox.use
 
 local lastbox = -1
 
-token.luacmd("saveboxresource", function(_, immediate)
+lmlt.luacmd("saveboxresource", function(_, immediate)
   if immediate == "value" then return end
-  if immediate and immediate & 0x7 ~= 0 then
+  if immediate and immediate & ~immediate_flag ~= 0 then
-    immediate = immediate & 0x8
+    immediate = immediate & immediate_flag
     tex.error("Unexpected prefix", "You used \\saveboxresource with a prefix that doesn't belong there. I will ignore it for now.")
   end
   local type
@@ -937,11 +959,11 @@ token.luacmd("saveboxresource", function(_, immediate)
   local margin = scan_keyword'margin' and scan_dimen() or nil
   local box = scan_int()
 
-  local index = tex.saveboxresource(box, attr, resources, immediate == 8, type, margin)
+  local index = tex.saveboxresource(box, attr, resources, immediate == immediate_flag, type, margin)
   lastbox = index
 end, "value")
 
-token.luacmd("useboxresource", function()
+lmlt.luacmd("useboxresource", function()
   local width, height, depth
   while true do
     if scan_keyword'width' then
@@ -958,7 +980,7 @@ token.luacmd("useboxresource", function()
   node.write((tex.useboxresource(index, width, height, depth)))
 end, "protected")
 
-token.luacmd("lastsavedboxresourceindex", function()
+lmlt.luacmd("lastsavedboxresourceindex", function()
   return integer_code, lastbox
 end, "value")
 
@@ -966,15 +988,15 @@ local saved_pos_x, saved_pos_y = -1, -1
 local save_pos_whatsit = declare_whatsit('save_pos', function(_, _, _, x, y)
   saved_pos_x, saved_pos_y = assert(math.tointeger(x)), assert(math.tointeger(y))
 end)
-token.luacmd("savepos", function() -- \savepos
+lmlt.luacmd("savepos", function() -- \savepos
   return node.direct.write(node.direct.new(whatsit_id, save_pos_whatsit))
 end, "protected")
 
-token.luacmd("lastxpos", function()
+lmlt.luacmd("lastxpos", function()
   return integer_code, (saved_pos_x+.5)//1
 end, "value")
 
-token.luacmd("lastypos", function()
+lmlt.luacmd("lastypos", function()
   return integer_code, (saved_pos_y+.5)//1
 end, "value")
 

luametalatex-basecallbacks.lua

@@ -4,6 +4,27 @@ local read_tfm = font.read_tfm
 local font_define = font.define
 local callbacks = require'luametalatex-callbacks'
 
+local find_file = kpse.find_file
+local output_directory = arg['output-directory']
+
+local dir_sep = '/' -- FIXME
+
+if output_directory then
+  local old_find_file = find_file
+  local kpse_absolute = kpse.is_absolute
+  local attributes = lfs.attributes
+  function find_file(path, kind, must_exists)
+    --kind is always "tex"
+    if not kpse_absolute(path) then
+      local new_path = output_directory .. dir_sep .. path
+      if attributes(new_path, 'mode') == 'file' then
+        return new_path
+      end
+    end
+    return old_find_file(path, kind, must_exists)
+  end
+end
+
 if status.ini_version then
   function callbacks.define_font(name, size)
     local f = read_tfm(name, size)
@@ -26,7 +47,11 @@ else
 end
 callbacks.__freeze'define_font'
 
+if output_directory then
+  function callbacks.find_log_file(name) return output_directory .. dir_sep .. name end
+else
   function callbacks.find_log_file(name) return name end
+end
 callbacks.__freeze'find_log_file'
 
 -- find_data_file is not an engine callback in luametatex, so we don't __freeze it
@@ -35,14 +60,14 @@ if status.ini_version then
     callbacks.find_data_file = nil
   end
   function callbacks.find_data_file(name)
-    if name == 'ltexpl.ltx' then
+    if name == 'expl3.ltx' then
       name = 'luametalatex-ltexpl-hook'
     end
-    return kpse.find_file(name, 'tex', true)
+    return find_file(name, 'tex', true)
   end
 end
   local function normal_find_data_file(name)
-    return kpse.find_file(name, 'tex', true)
+    return find_file(name, 'tex', true)
   end
 function callbacks.open_data_file(name)
   local find_callback = callbacks.find_data_file
@@ -50,7 +75,7 @@ function callbacks.open_data_file(name)
   if find_callback then
     path = find_callback(name)
   else
-    path = kpse.find_file(name, 'tex', true)
+    path = find_file(name, 'tex', true)
   end
   if not path then return end
 
@@ -99,18 +124,24 @@ local do_terminal_input do
   end
 end
 
+do
   local errorvalues = tex.geterrorvalues()
-function callbacks.intercept_tex_error(mode, errortype)
+  local function intercept(mode, errortype)
     errortype = errorvalues[errortype]
     if errortype == "eof" then
-    tex.runlocal(function()token.put_next(token.create'tracingall')end)
+      -- print('EOF', token.peek_next())
-    do_terminal_input()
+      token.put_next(token.create'ABD')
-    tex.runlocal(token.skip_next)
       return 3
     end
     texio.write'.'
+    if mode == 0 then return 0 end
+    local help = tex.gethelptext() or "Sorry, I don't know how to help in this situation.\n\z
+      Maybe you should try asking a human?"
     tex.showcontext()
-  if mode ~= 3 then return mode end
+    if mode ~= 3 then
+      texio.write('logfile', help .. '\n')
+      return mode
+    end
     repeat
       texio.write_nl'? '
       local line = io.read()
@@ -120,8 +151,7 @@ function callbacks.intercept_tex_error(mode, errortype)
       if line == "" then return 3 end
       local first = line:sub(1,1):upper()
       if first == 'H' then
-      texio.write(tex.gethelptext() or "Sorry, I don't know how to help in this situation.\n\z
+        texio.write(help)
-        Maybe you should try asking a human?")
       elseif first == 'I' then
         line = line:sub(2)
         tex.runlocal(function()
@@ -141,4 +171,14 @@ function callbacks.intercept_tex_error(mode, errortype)
     until false
     return 3
   end
+  function callbacks.intercept_tex_error(mode, errortype)
+    local ret = intercept(mode, errortype)
+    if tex.deadcycles >= tex.maxdeadcycles then
+      tex.runtoks(function()
+        tex.sprint(1, '\\shipout\\box\\outputbox')
+      end)
+    end
+    return ret
+  end
+end
 callbacks.__freeze'intercept_tex_error'

luametalatex-baseregisters.lua

@@ -1,9 +1,11 @@
+local lmlt = luametalatex
+
 local scan_dimen = token.scan_dimen
-local scan_int = token.scan_integer
+local scan_int = token.scan_int
 local scan_keyword = token.scan_keyword
 
-local count_code = token.value.integer
+local count_code = lmlt.value.integer
-local dimen_code = token.value.dimension
+local dimen_code = lmlt.value.dimension
 
 local set_local = require'luametalatex-local'
 
@@ -26,7 +28,7 @@ function texmeta.__newindex(t, k, v)
 end
 
 local function tex_variable(value, scanner, name, default)
-  token.luacmd(name, function(_, scanning)
+  lmlt.luacmd(name, function(_, scanning)
     if scanning == 'value' then
       return value, tex_variables[name]
     else
@@ -95,7 +97,7 @@ end
 
 local function pdf_variable(value, scanner, name, default, force_default)
   pdf_variable_names[#pdf_variable_names+1] = name
-  token.luacmd('pdfvariable  ' .. name, function(_, scanning)
+  lmlt.luacmd('pdfvariable  ' .. name, function(_, scanning)
     if scanning == 'value' then
       return value, real_pdf_variables[name]
     elseif force_default then
@@ -118,9 +120,15 @@ end
 tex_variable(count_code, scan_int, 'suppressfontnotfounderror', 0)
 tex_variable(count_code, scan_int, 'outputmode', 1) -- The "traditional" default would be 0,
                                                     -- but we do not actually support that.
+tex_variable(count_code, scan_int, 'mag', 1000)     -- Let's hope that noone wants to use these
+tex_variable(count_code, scan_int, 'tracingscantokens', 0)
+
 tex_variable(dimen_code, scan_dimen, 'pageheight', 0)
 tex_variable(dimen_code, scan_dimen, 'pagewidth', 0)
 
+tex_variable(dimen_code, scan_dimen, 'hoffset', 0)
+tex_variable(dimen_code, scan_dimen, 'voffset', 0)
+
 tex_variable(count_code, scan_int, 'bodydirection', 0)
 tex_variable(count_code, scan_int, 'pagedirection', 0)
 

luametalatex-dir-registers.lua

@@ -1,4 +1,4 @@
-local scan_int = token.scan_integer
+local scan_int = token.scan_int
 local scan_keyword = token.scan_keyword
 
 -- local names = {}
@@ -10,7 +10,7 @@ function tex.gettextdir() return tex.textdirection end
 function tex.getlinedir() return tex.linedirection end
 function tex.getmathdir() return tex.mathdirection end
 function tex.getpardir()  return tex.pardirection  end
-local integer_code = token.value.integer
+local integer_code = luametalatex.value.integer
 local function set_xdir(id, scanning)
   if scanning == 'value' then
     return integer_code, getters[id]()
@@ -30,7 +30,7 @@ return function(name)
   local getter = tex["get" .. name]
   local setter = tex["set" .. name]
   assert(getter and setter, "direction parameter undefined")
-  local idx = token.luacmd(name, set_xdir, "protected", "global", "value")
+  local idx = luametalatex.luacmd(name, set_xdir, "protected", "global", "value")
   getters[idx] = getter
   setters[idx] = setter
   return idx

luametalatex-firstcode.lua

@@ -1,7 +1,9 @@
-local scan_int = token.scan_integer
+local lmlt = luametalatex
+
+local scan_int = token.scan_int
 token.scan_int = scan_int -- For compatibility with LuaTeX packages
 local scan_token = token.scan_token
-local scan_tokenlist = token.scan_tokenlist
+local scan_tokenlist = token.scantokenlist
 local scan_keyword = token.scan_keyword
 local scan_csname = token.scan_csname
 local set_macro = token.set_macro
@@ -24,6 +26,35 @@ end
 --     end
 --   })
 
+do
+  luametalatex.texio = texio
+  local compat_texio = {}
+  for k,v in next, texio do compat_texio[k] = v end
+  local writenl = texio.writenl
+  local write = texio.write
+  texio = compat_texio
+  function texio.write(selector, ...)
+    if selector == 'term' then
+      selector = 'terminal'
+    elseif selector == 'log' then
+      selector = 'logfile'
+    elseif selector == 'term and log' then
+      selector = 'terminal_and_logfile'
+    end
+    return write(selector, ...)
+  end
+  function texio.write_nl(selector, ...)
+    if selector == 'term' then
+      selector = 'terminal'
+    elseif selector == 'log' then
+      selector = 'logfile'
+    elseif selector == 'term and log' then
+      selector = 'terminal_and_logfile'
+    end
+    return writenl(selector, ...)
+  end
+end
+
 local new_whatsit = require'luametalatex-whatsits'.new
 local whatsit_id = node.id'whatsit'
 local spacer_cmd, relax_cmd = token.command_id'spacer', token.command_id'relax'
@@ -61,31 +92,39 @@ if status.ini_version then
   setcatcode('global', 2, 32, 10)
 end
 
+local immediate_flag = lmlt.flag.immediate
+
 local l = lpeg or require'lpeg'
 local add_file_extension = l.Cs((1-('.' * (1-l.S'./\\')^0) * -1)^0 * (l.P(1)^1+l.Cc'.tex'))
 local ofiles, ifiles = {}, {}
+local output_directory = arg['output-directory']
+local dir_sep = '/' -- FIXME
 local function do_openout(p)
   if ofiles[p.file] then
     ofiles[p.file]:close()
   end
   local msg
-  ofiles[p.file], msg = io.open(add_file_extension:match(p.name), 'w')
+  local name = add_file_extension:match(p.name)
+  if output_directory then
+    name = output_directory .. dir_sep .. name
+  end
+  ofiles[p.file], msg = io.open(name, 'w')
   if not ofiles[p.file] then
     error(msg)
   end
 end
 local open_whatsit = new_whatsit('open', do_openout)
-token.luacmd("openout", function(_, immediate) -- \openout
+lmlt.luacmd("openout", function(_, immediate) -- \openout
   if immediate == "value" then return end
-  if immediate and immediate & 0x7 ~= 0 then
+  if immediate and immediate & ~immediate_flag ~= 0 then
-    immediate = immediate & 0x8
+    immediate = immediate & immediate_flag
     tex.error("Unexpected prefix", "You used \\openout with a prefix that doesn't belong there. I will ignore it for now.")
   end
   local file = scan_int()
   scan_keyword'='
   local name = scan_filename()
   local props = {file = file, name = name}
-  if immediate and immediate == 8 then
+  if immediate and immediate == immediate_flag then
     do_openout(props)
   else
     local whatsit = node.direct.new(whatsit_id, open_whatsit)
@@ -93,7 +132,7 @@ token.luacmd("openout", function(_, immediate) -- \openout
     node.direct.write(whatsit)
   end
 end, "value")
-token.luacmd("openin", function(_, prefix)
+lmlt.luacmd("openin", function(_, prefix)
   if prefix == "value" then return end
   local file = scan_int()
   scan_keyword'='
@@ -112,15 +151,15 @@ local function do_closeout(p)
   end
 end
 local close_whatsit = new_whatsit('close', do_closeout)
-token.luacmd("closeout", function(_, immediate) -- \closeout
+lmlt.luacmd("closeout", function(_, immediate) -- \closeout
   if immediate == "value" then return end
-  if immediate and immediate & 0x7 ~= 0 then
+  if immediate and immediate & ~immediate_flag ~= 0 then
-    immediate = immediate & 0x8
+    immediate = immediate & immediate_flag
     tex.error("Unexpected prefix", "You used \\closeout with a prefix that doesn't belong there. I will ignore it for now.")
   end
   local file = scan_int()
   local props = {file = file}
-  if immediate == 8 then
+  if immediate == immediate_flag then
     do_closeout(props)
   else
     local whatsit = node.direct.new(whatsit_id, close_whatsit)
@@ -128,7 +167,7 @@ token.luacmd("closeout", function(_, immediate) -- \closeout
     node.direct.write(whatsit)
   end
 end, "value")
-token.luacmd("closein", function(_, prefix)
+lmlt.luacmd("closein", function(_, prefix)
   if prefix == "value" then return end
   local file = scan_int()
   if ifiles[file] then
@@ -148,16 +187,16 @@ local function do_write(p)
   end
 end
 local write_whatsit = new_whatsit('write', do_write)
-token.luacmd("write", function(_, immediate) -- \write
+lmlt.luacmd("write", function(_, immediate, ...) -- \write
   if immediate == "value" then return end
-  if immediate and immediate & 0x7 ~= 0 then
+  if immediate and immediate & ~immediate_flag ~= 0 then
-    immediate = immediate & 0x8
+    immediate = immediate & immediate_flag
     tex.error("Unexpected prefix", "You used \\write with a prefix that doesn't belong there. I will ignore it for now.")
   end
   local file = scan_int()
   local content = scan_tokenlist()
   local props = {file = file, data = content}
-  if immediate == 8 then
+  if immediate == immediate_flag then
     do_write(props)
   else
     local whatsit = node.direct.new(whatsit_id, write_whatsit)
@@ -176,13 +215,13 @@ local function prefix_to_tokens(prefix)
     end
   end
 end
-local expand_after = token.primitive_tokens.expandafter
+local expand_after = lmlt.primitive_tokens.expandafter
-local input_tok = token.primitive_tokens.input
+local input_tok = lmlt.primitive_tokens.input
-local endlocalcontrol = token.primitive_tokens.endlocalcontrol
+local endlocalcontrol = lmlt.primitive_tokens.endlocalcontrol
-local afterassignment = token.primitive_tokens.afterassignment
+local afterassignment = lmlt.primitive_tokens.afterassignment
 local lbrace = token.new(0, 1)
 local rbrace = token.new(0, 2)
-token.luacmd("read", function(_, prefix)
+lmlt.luacmd("read", function(_, prefix)
   if immediate == "value" then return end
   local id = scan_int()
   if not scan_keyword'to' then
@@ -190,6 +229,9 @@ token.luacmd("read", function(_, prefix)
   end
   local macro = scan_csname(true)
   local file = ifiles[id]
+  local tokens = {}
+  local balance = 0
+  repeat
     local line
     if file then
       line = file:reader()
@@ -198,19 +240,17 @@ token.luacmd("read", function(_, prefix)
         ifiles[id] = nil
       end
     else
-    error[[FIXME: Ask the user for input]]
+      line = io.stdin:read()
     end
     local endlocal
     tex.runlocal(function()
-    endlocal = token.scan_next()
+      endlocal = token.get_next()
       tex.sprint(endlocal)
       tex.print(line and line ~= "" and line or " ")
       tex.print(endlocal)
     end)
-  local tokens = {}
-  local balance = 0
     while true do
-    local tok = token.scan_next()
+      local tok = token.get_next()
       if tok == endlocal then break end
       if tok.command == 1 then
         balance = balance + 1
@@ -219,16 +259,16 @@ token.luacmd("read", function(_, prefix)
       end
       tokens[#tokens+1] = tok
     end
-  if balance ~= 0 then error[[FIXME: Read additional input lines]] end
+  until balance == 0
   tex.runlocal(function()
     tokens[#tokens+1] = rbrace
     token.put_next(tokens)
-    token.put_next(token.primitive_tokens.def, token.create(macro), lbrace)
+    token.put_next(lmlt.primitive_tokens.def, token.create(macro), lbrace)
     prefix_to_tokens(prefix)
   end)
 end, "value")
 
-token.luacmd("readline", function(_, prefix)
+lmlt.luacmd("readline", function(_, prefix)
   if immediate == "value" then return end
   local id = scan_int()
   if not scan_keyword'to' then
@@ -265,7 +305,7 @@ local integer_code, boolean_code do
   end
 end
 
-token.luacmd("ifeof", function(_)
+lmlt.luacmd("ifeof", function(_)
   local id = scan_int()
   return boolean_code, not ifiles[id]
 end, "condition")
@@ -282,7 +322,7 @@ local late_lua_whatsit = new_whatsit('late_lua', function(p, pfile, n, x, y)
   end
   return pdf._latelua(pfile, x, y, code)
 end)
-token.luacmd("latelua", function() -- \latelua
+lmlt.luacmd("latelua", function() -- \latelua
   local content = scan_tokenlist()
   local props = {token = content}
   local whatsit = node.direct.new(whatsit_id, late_lua_whatsit)
@@ -297,7 +337,7 @@ require'luametalatex-baseregisters'
 require'luametalatex-back-pdf'
 require'luametalatex-node-luaotfload'
 
-token.luacmd("Umathcodenum", function(_, scanning)
+lmlt.luacmd("Umathcodenum", function(_, scanning)
   if scanning then
     local class, family, char = tex.getmathcodes (scan_int())
     return integer_code, char | (class | family << 3) << 21

luametalatex-font-resolve.lua

@@ -14,6 +14,10 @@ font.fonts = all_fonts
 function font.getfont(id)
   return all_fonts[id]
 end
+-- Currently either nil or true, false (defined but not yet frozen) isn't implemented right now
+function font.frozen(id)
+  return all_fonts[id] and true
+end
 
 local fontextensions = {
   ttf = "truetype",

luametalatex-font-t1.lua

@@ -40,8 +40,8 @@ local binary_bytes = lpeg.Cmt(number*white^-1*(lpeg.P'-| ' + 'RD '), function(s,
 
 local function decrypt(key, n, cipher)
   -- Generally you should never implement your own crypto. So we call a well known, peer reviewed,
-  -- high-quality cryptographic library. --- Ha-Ha, of course we are implementing by ourselves.
+  -- high-quality cryptographic library. --- Ha-Ha, of course we are implementing this by ourselves.
-  -- That might be completely unsecure, but given that the encryption keys are well known constants
+  -- That might be completely insecure, but given that the encryption keys are well known constants
   -- documented in the T1 Spec, there is no need to worry about it.
   -- Also I do not think any cryptographic library would implement this anyway, it doesn't even
   -- really deserve the term encryption.

luametalatex-font-t1tot2.lua

@@ -258,19 +258,22 @@ local function adjust_charstring(cs) -- Here we get a not yet optimized but pars
   local i = 1
   while cs[i] ~= nil do
     if cs[i] and cs[i].idx then
+      if stem3[2] then
+        local s3 = stem3[2]
+        for l = 1, math.floor((j + 7)/8) do
+          bytes[l] = string.byte(s3, l)
+        end
+      else
         for l = 1, math.floor((j + 7)/8) do
           bytes[l] = 0
         end
+      end
       while (cs[i] or {}).idx do
         local idx = cs[i].idx-1
         bytes[math.floor(idx/8) + 1] = bytes[math.floor(idx/8) + 1] | (1<<(7-idx%8))
         cs[i] = false
         i = i+1
       end
-      for l = 2, #stem3 do
-        local idx = stem3[l].idx-1
-        bytes[math.floor(idx/8) + 1] = bytes[math.floor(idx/8) + 1] | (1<<(7-idx%8))
-      end
       i = i-1
       cs[i] = {19, string.char(table.unpack(bytes))}
     end

luametalatex-init.lua

@@ -24,19 +24,6 @@ do
   end
 end
 kpse.set_program_name(arg.arg0 or arg[arg[0]], arg.progname)
-do
-  local interaction = ({ [true] = 3, [false] = false,
-    batchmode=0,
-    nonstopmode=1,
-    scrollmode=2,
-    errorstopmode=3,
-  })[arg.interaction or false]
-  if interaction then
-    tex.setinteraction(interaction)
-  elseif interaction == nil then
-    texio.write('term', string.format('Unknown interaction mode %q ignored.\n', arg.interaction))
-  end
-end
 package.searchers[2] = function(modname)
   local filename = kpse.find_file(modname, "lua", true)
   if not filename then
@@ -72,6 +59,19 @@ end
 callback_register('find_format_file', function(name) texconfig.formatname = kpse.find_file(name, 'fmt', true) return texconfig.formatname end)
 -- texconfig.firstline = [[\show ]]
 function texconfig.init()
+  do
+    local interaction = ({ [true] = 3, [false] = false,
+      batchmode=0,
+      nonstopmode=1,
+      scrollmode=2,
+      errorstopmode=3,
+    })[arg.interaction or false]
+    if interaction then
+      tex.setinteraction(interaction)
+    elseif interaction == nil then
+      texio.write('term', string.format('Unknown interaction mode %q ignored.\n', arg.interaction))
+    end
+  end
   if build_bytecode then -- Effectivly if status.ini_version
     require'luametalatex-lateinit'(build_bytecode)
   else

luametalatex-lateinit.lua

@@ -1,3 +1,5 @@
+luametalatex = luametalatex or {}
+local lmlt = luametalatex
 local initex = status.ini_version
 
 if initex then
@@ -14,6 +16,7 @@ pdf = {
 }
 require'luametalatex-font-resolve' -- Replace font.define. Must be loaded before callbacks
 require'luametalatex-basecallbacks'
+require'luametalatex-oldnames'
 local callbacks = require'luametalatex-callbacks'
 
 local function swap_table(t)
@@ -33,13 +36,14 @@ do
     primitives[prim[3]] = token_new(prim[2], prim[1])
   end
 end
-token.primitive_tokens = primitives
+lmlt.primitive_tokens = primitives
 
 do
   local command_id = swap_table(token.getcommandvalues())
   function token.command_id(name) return command_id[name] end
 end
-token.value = swap_table(token.getfunctionvalues())
+lmlt.value = swap_table(token.getfunctionvalues())
+lmlt.flag = swap_table(tex.getflagvalues())
 
 local functions = lua.getfunctionstable()
 -- I am not sure why this is necessary, but otherwise LuaMetaTeX resets
@@ -68,9 +72,9 @@ end
 local undefined_cmd = token.command_id'undefined_cs'
 local lua_call_cmd = token.command_id'lua_call'
 local lua_value_cmd = token.command_id'lua_value'
-local lua_expandable_call_cmd = token.command_id'lua_expandable_call'
+local lua_protected_call_cmd = token.command_id'lua_protected_call'
 local if_test_cmd = token.command_id'if_test'
-function token.luacmd(name, func, ...)
+function lmlt.luacmd(name, func, ...)
   local idx
   local tok = token.create(name)
   local cmd = tok.command
@@ -78,10 +82,10 @@ function token.luacmd(name, func, ...)
     idx = tok.index
   elseif cmd == lua_call_cmd then
     idx = tok.index
-  elseif cmd == lua_expandable_call_cmd then
+  elseif cmd == lua_protected_call_cmd then
     idx = tok.index
-  elseif cmd == if_test_cmd and tok.index > 48 then
+  elseif cmd == if_test_cmd and tok.index > 53 then
-    idx = tok.index - 48
+    idx = tok.index - 53
   elseif ... == 'force' then
     idx = new_luafunction(name)
     set_lua(name, idx, select(2, ...))
@@ -100,15 +104,14 @@ end
 
 if initex then
   local build_bytecode = nil -- To be filled
+  local output_directory = arg['output-directory']
   function callbacks.pre_dump()
     local user_callback = callbacks.pre_dump
     if user_callback then user_callback() end
 
     local prepared = lua.prepared_code
     prepared[1] = string.format("fixupluafunctions(%i)", predefined_luafunctions)
-    for i=0,0 do -- maybeFIXME: In practise only one language is preloaded in LuaTeX anyway
+    for i=0,status.languagestate.ptr do
-    -- for i=0,tex.count[19] do -- Sometimes catches reserved language ids which are not used yet
-    -- for i=0,lang.new():id()-1 do -- lang.new():id() is always 0 in luametatex?!?
       local l = language.new(i)
       local str = string.format("do \z
         local l = language.new(%i)\z
@@ -142,6 +145,9 @@ if initex then
       end
     end
     lua.bytecode[tex.count[262]+1] = build_bytecode(table.concat(prepared, '\n'))
+    if output_directory then
+      lfs.chdir(output_directory) -- We can't change the location TeX writes it's format to, so we change the current directory instead
+    end
   end
   callbacks.__freeze('pre_dump', true)
   return function(f)

luametalatex-local.lua

@@ -12,7 +12,7 @@ lua.get_functions_table()[restore_func] = function()
     end
   end
 end
-local restore_toks = {token.primitive_tokens.atendofgroup , token.new(restore_func, token.command_id'lua_call')}
+local restore_toks = {luametalatex.primitive_tokens.atendofgroup , token.new(restore_func, token.command_id'lua_call')}
 local put_next = token.put_next
 local runlocal = tex.runlocal
 local function put_restore_toks()

luametalatex-ltexpl-hook.tex

@@ -2,10 +2,14 @@
 % See baseregisters for list of toks pdfvariables
 \directlua{initialize_pdf_toks()}
 % We hardcode the ids of two catcodetables. If they ever change in the format, abort.
-\if 0\ifnum1=\catcodetable@initex\else\expandafter1\fi\ifnum2=\catcodetable@string\else\expandafter1\fi 0\else
+\if 0\ifnum1=\catcodetable@initex\else1\fi\ifnum2=\catcodetable@string\else1\fi 0\else
   \errmessage{Inconsistent catcodetable identifiers}
 \fi
 \ifx\@tfor\undefined
   \def\@tfor#1\do#2{}
 \fi
-\input ltexpl.ltx
+\expanded {\def \noexpand \newlanguage #1\expandafter}\expandafter {%
+  \newlanguage{#1}%
+  \directlua{language.new(tex.count.allocationnumber)}%
+}%
+\input expl3.ltx

luametalatex-meaning.lua

@@ -0,0 +1,31 @@
+-- Compatibility layer for \meaning
+-- This reverts a change rearding the \meaning of user macros:
+-- LuaMetaTeX does not prefix prefixes in the meaning with a backslash.
+
+-- Necessary especially for etoolbox's patching commands
+
+local lmlt = luametalatex
+local primitive_meaning = {
+  lmlt.primitive_tokens.expandafter,
+  lmlt.primitive_tokens.relax,
+  lmlt.primitive_tokens.meaning,
+}
+local call_cmd = token.command_id'call'
+lmlt.luacmd("meaning", function()
+  local peeked = token.peeknext()
+  token.put_next(primitive_meaning)
+  token.skipnextexpanded()
+  if peeked.parameters == 0 or (peeked.command == call_cmd and not peeked.parameters) then
+    local stash = ''
+    if peeked.protected then
+      if token.scankeywordcs'protected' then
+        token.skipnext()
+        token.skipnext()
+        stash = stash .. '\\protected '
+      end
+    end
+    token.scankeyword'macro:'
+    tex.sprint(-2, stash)
+    tex.sprint(-2, 'macro:->')
+  end
+end, "force", "global")

luametalatex-microtype.sty

@@ -24,10 +24,10 @@
   \noexpand\luametalatex@@everyjobandnow{\noexpand\directlua{!
     lua.get_functions_table()[\the\luametalatex@@expandglyphsinfont] = function()
       token.put_next(token.create'fontid')
-      local font = token.scan_integer()
+      local font = token.scan_int()
-      local stretch = token.scan_integer()
+      local stretch = token.scan_int()
-      local shrink = token.scan_integer()
+      local shrink = token.scan_int()
-      local step = token.scan_integer()
+      local step = token.scan_int()
       token.set_macro('pickup@font@@hook@luametalatex@microtype@' .. font, string.format("{}{%i}{%i}{%i}", stretch, shrink, step), "global")
     end
   }}

luametalatex-nodewriter.lua

@@ -43,6 +43,7 @@ local get_whatsit_handler = require'luametalatex-whatsits'.handler
 local write_matrix -- Defined later
 
 local dir_id = node.id'dir'
+local glue_id = node.id'glue'
 
 local function doublekeyed(t, id2name, name2id, index)
   return setmetatable(t, {
@@ -273,7 +274,7 @@ nodehandler.math = ignore_node
 nodehandler.kern = ignore_node
 -- The following are only for frontend use:
 nodehandler.boundary = ignore_node
-nodehandler.local_par = ignore_node
+nodehandler.par = ignore_node
 nodehandler.penalty = ignore_node
 nodehandler.mark = ignore_node
 
@@ -356,7 +357,7 @@ function nodehandler.vlist(p, list, x, y0, outerlist, origin, level)
   local y = y0
   y = y + height
   for n in traverse(getlist(list)) do
-    local d, h, _ = 0, direct.effective_glue(n, list) or math.tointeger(getkern(n))
+    local d, h, _ = 0, getid(n) == glue_id and direct.effective_glue(n, list) or math.tointeger(getkern(n))
     if not h then
       _, h, d = direct.getwhd(n)
     end
@@ -625,12 +626,14 @@ vf = {
 }
 function nodehandler.glyph(p, n, x, y, outer, x0, level, direction)
   if getfont(n) ~= p.vfont.fid then
-    p.vfont.fid = getfont(n)
+    local fid = getfont(n)
-    p.vfont.font = font.getfont(getfont(n)) or font.fonts[getfont(n)]
+    p.vfont.fid = fid
+    p.vfont.font = font.fonts[fid]
   end
   local f, fid = p.vfont.font, p.vfont.fid
   local cid = getchar(n)
-  local c = f.characters[cid]
+  local c = f and f.characters
+  c = c and c[cid]
   if not c then
     texio.write_nl("Missing character")
     return
@@ -703,8 +706,10 @@ function pdf.write(mode, text, x, y, p)
     topage(p)
     p.strings[#p.strings + 1] = text
   elseif mode == "direct" then
-    if p.mode ~= page then
+    if p.mode < page then
       totext(p, p.font.fid)
+    elseif p.mode > page then
+      topage(p)
     end
     p.strings[#p.strings + 1] = text
   elseif mode == "origin" then

luametalatex-oldnames.lua

@@ -0,0 +1,124 @@
+local function rewrite(t)
+  local function handler(old, new)
+    t[old] = t[old] or t[new]
+    if not t[old] then print('!!!', old) end
+    return handler
+  end
+  return handler
+end
+
+rewrite(tex)
+  ('runtoks', 'runlocal')
+
+rewrite(language)
+  ('clear_hyphenation', 'clearhyphenation')
+  ('clear_patterns', 'clearpatterns')
+
+rewrite(node)
+  ('copy_list', 'copylist')
+  ('flush_list', 'flushlist')
+  ('flush_node', 'flushnode')
+  ('get_properties_table', 'getpropertiestable')
+  ('has_field', 'hasfield')
+  ('insert_after', 'insertafter')
+  ('insert_before', 'insertbefore')
+  ('is_node', 'isnode')
+
+rewrite(node.direct)
+  ('copy_list', 'copylist')
+  ('effective_glue', 'effectiveglue')
+  ('end_of_math', 'endofmath')
+  ('first_glyph', 'firstglyph')
+  ('flush_list', 'flushlist')
+  ('flush_node', 'flushnode')
+  ('get_attribute', 'getattribute')
+  ('get_properties_table', 'getpropertiestable')
+  ('has_field', 'hasfield')
+  ('has_glyph', 'hasglyph')
+  ('insert_after', 'insertafter')
+  ('insert_before', 'insertbefore')
+  ('is_char', 'ischar')
+  ('is_direct', 'isdirect')
+  ('is_glyph', 'isglyph')
+  ('is_node', 'isnode')
+  ('is_zero_glue', 'iszeroglue')
+  ('last_node', 'lastnode')
+  ('make_extensible', 'makeextensible')
+  ('mlist_to_hlist', 'mlisttohlist')
+  ('protect_glyph', 'protectglyph')
+  ('protect_glyphs', 'protectglyphs')
+  ('traverse_char', 'traversechar')
+  ('traverse_list', 'traverselist')
+  ('traverse_id', 'traverseid')
+  ('unprotect_glyph', 'unprotectglyph')
+  ('unprotect_glyphs', 'unprotectglyphs')
+  ('uses_font', 'usesfont')
+
+rewrite(token)
+  ('get_next', 'scannext')
+  ('put_next', 'putnext')
+  ('scan_argument', 'scanargument')
+  ('scan_box', 'scanbox')
+  ('scan_code', 'scancode')
+  ('scan_csname', 'scancsname')
+  ('scan_dimen', 'scandimen')
+  ('scan_float', 'scanfloat')
+  ('scan_glue', 'scanglue')
+  ('scan_int', 'scaninteger')
+  ('scan_keyword', 'scankeyword')
+  ('scan_keywordcs', 'scankeywordcs')
+  ('scan_real', 'scanreal')
+  ('scan_string', 'scanstring')
+  ('scan_token', 'scannextexpanded')
+  ('scan_toks', 'scantoks')
+  ('scan_word', 'scanword')
+  ('set_lua', 'setlua')
+  ('set_macro', 'setmacro')
+  ('set_char', 'setchar')
+  ('is_defined', 'isdefined')
+
+do
+  lang = {}
+  for k,v in next, language do
+    lang[k] = v
+  end
+  local patterns = language.patterns
+  function lang.patterns(l, patt)
+    if not patt then
+      return patterns(l)
+    end
+    patt = patt:gsub('%s', ' ')
+    return patterns(l, patt)
+  end
+end
+
+do
+  local l = lpeg or require'lpeg'
+  local match = l.match
+  local P = l.P
+  local C = l.C
+  local sub = string.sub
+  local any = C(P(1)^0)
+  local char = C(1)
+  function string.explode(s, delim)
+    delim = delim or ' +'
+    if s == '' then
+      return {s}
+    else
+      local patt
+      if delim == '' then -- Split into individual characters
+        patt = char
+      else
+        local del = P(sub(delim, 1, 1))
+        if sub(delim, 2, 2) == '+' then
+          local elem = C((1-del)^1)
+          patt = del^0 * elem * (del^1 * elem)^0
+        else
+          local elem = C((1-del)^0)
+          patt = elem * (del * elem)^0
+        end
+      end
+      return {match(patt, s)}
+    end
+  end
+end

luametalatex.ini

@@ -16,4 +16,29 @@
   \global\def\eTeXrevision{.2}
 \endgroup
 
+\let\LMLTsaveddump\dump
+\let\dump\relax
 \input latex.ltx
+\let\dump\LMLTsaveddump
+%
+\makeatletter
+\def\set@fontsize#1#2#3{%
+  \@defaultunits\@tempdimb#2pt\relax\@nnil
+  \edef\f@size{\strip@pt\@tempdimb}%
+  \@defaultunits\@tempdimc#3pt\relax\@nnil
+  \edef\f@baselineskip{\the\@tempdimc}%
+  \edef\f@linespread{#1}%
+  \let\baselinestretch\f@linespread
+  \def\size@update{%
+    \baselineskip\f@baselineskip\relax
+    \baselineskip\f@linespread\baselineskip
+    \normalbaselineskip\baselineskip
+    \setbox\strutbox\hbox{%
+      \vrule\@height.7\baselineskip
+            \@depth.3\baselineskip
+            \@width\z@}%
+    \let\size@update\relax}%
+}
+\makeatother
+%
+\dump

luametaplain.ini

@@ -17,5 +17,20 @@
 \endgroup
 \input load-unicode-data.tex
 \input etex.src
-\directlua{initialize_pdf_toks()}
+\directlua{initialize_pdf_toks()}%
+%
+\expanded {\def \noexpand \newlanguage #1\expandafter}\expandafter {%
+  \newlanguage{#1}%
+  \directlua{language.new(tex.count.allocationnumber)}%
+}%
+%
+\expanded {\def \noexpand \uselanguage #1\expandafter}\expandafter {%
+  \expanded {%
+    \noexpand \ifcsname lang@#1\endcsname
+      \noexpand \directlua {lang.new(\noexpand\number\noexpand\lastnamedcs)}%
+    \noexpand \fi
+    \expandafter
+  }%
+  \uselanguage{#1}%
+}%
 \dump

luametatexconfig.tex

@@ -31,7 +31,24 @@
   % so can now be read using the (emulated) primitives
   % This needs to be global so set \globaldefs for the rest of the group
   \globaldefs=1 %
-  \input{pdftexconfig}%
+  % \input{pdftexconfig}%
+  % Copied in here because the original uses `true` dimensions which aren't supported
+  \pdfoutput           = 1
+
+  % Paper size: dimensions given in absolute terms
+  \pdfpageheight       = 297 mm
+  \pdfpagewidth        = 210 mm
+
+  % Enable PDF 1.5 output and thus more compression
+  \pdfminorversion     = 5
+  \pdfobjcompresslevel = 2
+
+  % Low-level settings unlikely ever to need to change
+  \pdfcompresslevel    = 9
+  \pdfdecimaldigits    = 3
+  \pdfpkresolution     = 600
+  \pdfhorigin          = 1 in
+  \pdfvorigin          = 1 in
   % Pick up on a request for DVI mode and apply it whilst \pdfoutput is
   % still defined
   \ifx\dvimode\relax