Various adaptions

Gives better hinted output for asian fonts

.gitignore

@@ -0,0 +1 @@
+/build/

README.md

@@ -4,21 +4,21 @@
 This code is in early stages of development and contains more bugs than features. _Do not_ expect it to be compatible with normal documents. Also later versions will contain many breaking changes.
 
 ## Prerequisites
-You need an up-to-date TeX Live installation and the latest version of LuaMetaTeX.
+You need an up-to-date TeX Live installation and the latest version of LuaMetaTeX. Additionally a current development version of luaotfload is required. (Of course, luaotfload is not necessary if you only want to use `luametaplain` without Unicode fonts.)
 
-Additionally a special library version of LuaTeX's kpathsea Lua binding is needed which is provided as a binary for Linux x64. For other platforms you might have to compile it yourself. Drop me a line if you need any instructions. (The source can be found under https://github.com/zauguin/luametalatex-kpse)
+IF you are not using Linux x64 or Windows x64, you additionally need to compile a special library version of LuaTeX's kpathsea Lua binding. The source is available under https://github.com/zauguin/luametalatex-kpse . After compiling, the resulting `kpse.so` or `kpse.dll` has to be stored in the main directory of luametalatex (the same directory where `luametalatex-init.lua` is stored)
 
 ## How to install (automatically)
 Obtain `luametatex` from ConTeXt, drop the binary into the same location where your `luatex` binary is installed and then run `install.sh`.
 
 ## How to install (manually)
-Obtain `luametatex` from ConTeXt, drop the binary into the same location where your `luatex` binary is installed and copy (or sym-link) the file `luametalatex.lua` into the same directory. Additionally create a sym-link `luametalatex` to `luametatex` in the same directory. Then copy (or sym-link) this entire repo to `.../texmf-local/tex/lualatex/luametalatex`. 
+Obtain `luametatex` from ConTeXt, drop the binary into the same location where your `luatex` binary is installed and copy (or sym-link) the file `luametalatex.lua` into the same directory. Additionally create a sym-link `luametalatex` to `luametatex` in the same directory. Then install `luametalatex` into your texmf tree by running `l3build install` in this repository.
 
-Finally add the line
+Finally add the lines (on Windows, `$(luametalatex.lua)` has to be replaced by the full path to `luametalatex.lua`)
 ```
 luametalatex luametatex language.dat,language.dat.lua --lua="$(kpsewhich luametalatex.lua)" luametalatex.ini
 ```
-to your local `fmtutil.cnf` and configure paths for luametalatex in your `texmf.cnf`. Then you should be able to run `fmtutil-sys --byfmt luametalatex` to generate the format.
+to your local `fmtutil.cnf` and configure paths for luametalatex in your `texmf.cnf`. Then you should be able to run `mktexlsr` followed by `fmtutil-sys --byfmt luametalatex` to generate the format.
 
 If this worked you can built (simple) LaTeX documents using the command `luametalatex`.
 

build.lua

@@ -0,0 +1,8 @@
+#!/use/bin/env texlua
+
+module = "luametalatex"
+
+tdsroot = "luametatex" -- Would be luametalatex but we use the same files for luametaplain
+
+installfiles = {"luameta*.lua", "luameta*.ini", "*.so", "*.dll", "*texconfig.tex", "luametalatex-ltexpl-hook.tex"}
+sourcefiles = {"luameta*.lua", "luameta*.ini", "*.so", "*.dll", "*texconfig.tex", "luametalatex-ltexpl-hook.tex"}

install.sh

@@ -1,45 +1,42 @@
 #!/bin/sh
-ENGINE="$(which luametatex)"
+luametatex --credits >/dev/null || exit 1
+l3build install
+ENGINE="$(which luametatex$EXE_EXT)"
 ENGINE_DIR="$(dirname "$ENGINE")"
 REPO="$(pwd)"
 cd "$(dirname "$ENGINE")"
-ln -s luametatex luametaplain
-ln -s luametatex luametalatex
-ln -s luametatex luametalatex-dev
-ln -s "$REPO/luametaplain.lua" .
-ln -s "$REPO/luametalatex.lua" .
-ln -s "$REPO/luametalatex-dev.lua" .
-while [ ! -d texmf ] && [ ! -d texmf-local ]
-do
-  LASTDIR="$(pwd)"
-  cd ..
-  if [ "$(pwd)" == "$LASTDIR" ]
-  then
-    exit 1
-  fi
-done
-if [ -d texmf ]
-then cd texmf
-else cd texmf-local
-fi
-mkdir -p tex/luameta{plain,latex{,-dev}}
-ln -s "$REPO" tex/luametaplain/base
-ln -s "$REPO" tex/luametalatex/base
-ln -s "$REPO" tex/luametalatex-dev/base
+ln -fs luametatex$EXE_EXT luametaplain$EXE_EXT
+ln -fs luametatex$EXE_EXT luametalatex$EXE_EXT
+ln -fs luametatex$EXE_EXT luametalatex-dev$EXE_EXT
+ln -fs "$REPO/luametaplain.lua" luametaplain.lua
+ln -fs "$REPO/luametalatex.lua" luametalatex.lua
+ln -fs "$REPO/luametalatex-dev.lua" luametalatex-dev.lua
+cd "$(kpsewhich -var-value TEXMFLOCAL)"
 mkdir -p web2c
-cat >> web2c/texmf.cnf << "EOF"
-TEXINPUTS.luametaplain    = $TEXMFDOTDIR;$TEXMF/tex/{luametatex,luatex,plain,generic,}//
+if kpsewhich -var-value LUAINPUTS.luametalatex > /dev/null
+then
+  echo 'LUAINPUTS for luametalatex already set. In case of issues, please verify that the entries are correct.'
+else
+  cat >> web2c/texmf.cnf << "EOF"
+TEXINPUTS.luametaplain    = $TEXMFDOTDIR;$TEXMF/tex/{luametaplain,luametatex,luatex,plain,generic,}//
 TEXINPUTS.luametalatex    = $TEXMFDOTDIR;$TEXMF/tex/{luametalatex,lualatex,latex,luametatex,luatex,generic,}//
 TEXINPUTS.luametalatex-dev= $TEXMFDOTDIR;$TEXMF/tex/{luametalatex,latex-dev,lualatex,latex,luametatex,luatex,generic,}//
 
-LUAINPUTS.luametaplain    = $TEXMFDOTDIR;$TEXMF/scripts/{$progname,$engine,}/{lua,}//;$TEXMF/tex/{luametatex,luatex,plain,generic,}//
+LUAINPUTS.luametaplain    = $TEXMFDOTDIR;$TEXMF/scripts/{$progname,$engine,}/{lua,}//;$TEXMF/tex/{luametaplain,luametatex,luatex,plain,generic,}//
 LUAINPUTS.luametalatex    = $TEXMFDOTDIR;$TEXMF/scripts/{$progname,$engine,}/{lua,}//;$TEXMF/tex/{luametalatex,lualatex,latex,luametatex,luatex,generic,}//
 LUAINPUTS.luametalatex-dev= $TEXMFDOTDIR;$TEXMF/scripts/{$progname,$engine,}/{lua,}//;$TEXMF/tex/{luametalatex,latex-dev,lualatex,latex,luametatex,luatex,generic,}//
 EOF
+fi
 
-cat >> web2c/fmtutil.cnf << "EOF"
-luametaplain luametatex language.dat,language.dat.lua --lua="$(kpsewhich luametalatex.lua)" luametaplain.ini
+if fmtutil-user --listcfg|grep -q '^luametalatex '
+then
+  echo 'luametalatex format already known. In case of issues, please verify that the entries are correct.'
+else
+  cat >> web2c/fmtutil.cnf << "EOF"
+luametaplain luametatex language.dat,language.dat.lua --lua="$(kpsewhich luametaplain.lua)" luametaplain.ini
 luametalatex luametatex language.dat,language.dat.lua --lua="$(kpsewhich luametalatex.lua)" luametalatex.ini
 luametalatex-dev luametatex language.dat,language.dat.lua --lua="$(kpsewhich luametalatex-dev.lua)" luametalatex.ini
 EOF
+fi
+
 echo INSTALLED

luametalatex-back-pdf.lua

@@ -1,6 +1,16 @@
+local scan_int = token.scan_integer
+local scan_token = token.scan_token
+local scan_keyword = token.scan_keyword
+local scan_string = token.scan_string
+local scan_word = token.scan_word
+local scan_dimen = token.scan_dimen
+local scan_box = token.scan_box
+token.scan_list = scan_box -- They are equal if no parameter is present
+
 local pdf = pdf
 local pdfvariable = pdf.variable
 
+local callbacks = require'luametalatex-callbacks'
 local writer = require'luametalatex-nodewriter'
 local newpdf = require'luametalatex-pdf'
 local nametree = require'luametalatex-pdf-nametree'
@@ -36,7 +46,6 @@ local colorstacks = {{
     default = "0 g 0 G",
     page_stack = {"0 g 0 G"},
   }}
-token.scan_list = token.scan_box -- They are equal if no parameter is present
 local spacer_cmd = token.command_id'spacer'
 local function get_pfile()
   if not pfile then
@@ -68,29 +77,62 @@ local reset_deadcycles = {
 }
 token.luacmd("shipout", function()
   local pfile = get_pfile()
+  local total_voffset, total_hoffset = tex.voffset + pdfvariable.vorigin, tex.hoffset + pdfvariable.horigin
   local voff = node.new'kern'
-  voff.kern = tex.voffset + pdfvariable.vorigin
-  voff.next = token.scan_list()
-  voff.next.shift = tex.hoffset + pdfvariable.horigin
+  voff.kern = total_voffset
+  voff.next = scan_box()
+  voff.next.shift = total_hoffset
   local list = node.direct.tonode(node.direct.vpack(node.direct.todirect(voff)))
-  list.height = tex.pageheight
-  list.width = tex.pagewidth
+  local pageheight, pagewidth = tex.pageheight, tex.pagewidth
+  -- In the following, the total_[hv]offset represents a symmetric offset applied on the right/bottom.
+  -- The upper/left one is already included in the box dimensions
+  list.height = pageheight ~= 0 and pageheight or list.height + list.depth + total_voffset
+  list.width = pagewidth ~= 0 and pagewidth or list.width + total_hoffset
   local page, parent = pfile:newpage()
   cur_page = page
   local out, resources, annots = writer(pfile, list, fontdirs, usedglyphs, colorstacks)
   cur_page = nil
   local content = pfile:stream(nil, '', out)
-  pfile:indirect(page, string.format([[<</Type/Page/Parent %i 0 R/Contents %i 0 R/MediaBox[0 %i %i %i]/Resources%s%s%s>>]], parent, content, -math.ceil(to_bp(list.depth)), math.ceil(to_bp(list.width)), math.ceil(to_bp(list.height)), resources(pdfvariable.pageresources), annots, pdfvariable.pageattr))
+  pfile:indirect(page, string.format([[<</Type/Page/Parent %i 0 R/Contents %i 0 R/MediaBox[0 %i %i %i]/Resources%s%s%s%s>>]], parent, content, -math.ceil(to_bp(list.depth)), math.ceil(to_bp(list.width)), math.ceil(to_bp(list.height)), resources(pdfvariable.pageresources .. pdf.pageresources), annots, pdfvariable.pageattr, pdf.pageattributes))
   node.flush_list(list)
   token.put_next(reset_deadcycles)
-  token.scan_token()
+  scan_token()
 end, 'force', 'protected')
 
 local infodir = ""
 local namesdir = ""
 local catalogdir = ""
 local catalog_openaction
-local creationdate = os.date("D:%Y%m%d%H%M%S%z"):gsub("+0000$", "Z"):gsub("%d%d$", "'%0")
+local creationdate = os.date("D:%Y%m%d%H%M%S")
+do
+  local time0 = os.time()
+  local tz = os.date('%z', time0)
+  if tz:match'^[+-]%d%d%d%d$' then
+    if tz:sub(1) == '0000' then
+      tz = 'Z'
+    else
+      tz = tz:sub(1,3) .. "'" .. tz:sub(4)
+    end
+  else
+    local utc_time = os.date('!*t')
+    utc_time.isdst = nil
+    local time1 = os.time(utc_time)
+    local offset = time1-time0
+    if offset == 0 then
+      tz = 'Z'
+    else
+      if offset > 0 then
+        tz = '-'
+      else
+        tz = '+'
+        offset = -offset
+      end
+      offset = offset // 60
+      tz = string.format("%s%02i'%02i", tz, offset//60, offset%60)
+    end
+  end
+  creationdate = creationdate .. tz
+end
 local function write_infodir(p)
   local additional = ""
   if not string.find(infodir, "/CreationDate", 1, false) then
@@ -106,7 +148,7 @@ local function write_infodir(p)
     additional = string.format("%s/Creator(TeX)", additional)
   end
   if not string.find(infodir, "/PTEX.Fullbanner", 1, false) then
-    additional = string.format("%s/PTEX.Fullbanner(%s)", additional, status.banner)
+    additional = string.format("%s/PTEX.Fullbanner(%s)", additional, status.enginestate.banner)
   end
   return p:indirect(nil, string.format("<<%s%s>>", infodir, additional))
 end
@@ -120,7 +162,10 @@ local function nodefont_newindex(t, k, v)
   return rawset(t, k, v)
 end
 
-callback.register("stop_run", function()
+function callbacks.stop_run()
+  local user_callback = callbacks.stop_run
+  if user_callback then user_callback() end
+
   if not pfile then
     return
   end
@@ -181,7 +226,7 @@ callback.register("stop_run", function()
   local size = pfile:close()
   texio.write_nl("term", "(see the transcript file for additional information)")
   -- TODO: Additional logging, epecially targeting the log file
-  texio.write_nl("term and log", string.format(" %d words of node memory still in use:", status.var_used))
+  texio.write_nl("term and log", string.format(" %d words of node memory still in use:", status.nodestate.use))
   local by_type, by_sub = {}, {}
   for n, id, sub in node.traverse(node.usedlist()) do
     if id == whatsit_id then
@@ -200,20 +245,22 @@ callback.register("stop_run", function()
   end
   texio.write_nl("  " .. table.concat(nodestat, ', '))
   texio.write_nl(string.format("Output written on %s (%d pages, %d bytes).", pdfname, pages, size))
-  texio.write_nl(string.format("Transcript written on %s.\n", status.log_name))
-end, "Finish PDF file")
+  texio.write_nl(string.format("Transcript written on %s.\n", status.enginestate.logfilename))
+end
+callbacks.__freeze('stop_run', true)
+
 token.luacmd("pdfvariable", function()
   for _, n in ipairs(pdf.variable_names) do
-    if token.scan_keyword(n) then
+    if scan_keyword(n) then
       return token.put_next(token.create('pdfvariable  ' .. n))
     end
   end
   -- The following error message gobbles the next word as a side effect.
   -- This is intentional to make error-recovery easier.
   --[[
-  error(string.format("Unknown PDF variable %s", token.scan_word()))
+  error(string.format("Unknown PDF variable %s", scan_word()))
   ]] -- Delay the error to ensure luatex85.sty compatibility
-  texio.write_nl(string.format("Unknown PDF variable %s", token.scan_word()))
+  texio.write_nl(string.format("Unknown PDF variable %s", scan_word()))
   tex.sprint"\\unexpanded{\\undefinedpdfvariable}"
 end)
 
@@ -250,33 +297,29 @@ local function get_action_attr(p, action, is_link)
   if action_type == 2 then
     error[[FIXME: Threads are currently unsupported]] -- TODO
   elseif action_type == 0 then
-    error[[FIXME]]
-  elseif action_type == 1 then -- GoTo
-    local id = action.id
-    if id then
-      if file then
-        assert(type(id) == "string")
-        action_attr = action_attr .. "/S/GoToR/D" .. pdf_bytestring(id) .. ">>"
-      else
-        local dest = dests[id]
-        if not dest then
-          dest = pfile:getobj()
-          dests[id] = dest
-        end
-        if type(id) == "string" then
-          action_attr = action_attr .. "/S/GoTo/D" .. pdf_bytestring(id) .. ">>"
-        else
-          action_attr = string.format("%s/S/GoTo/D %i 0 R>>", action_attr, dest)
-        end
-      end
+    local page = assert(action.page, 'Page action must contain a page')
+    local tokens = action.tokens
+    if file then
+      action_attr = string.format("%s/S/GoToR/D[%i %s]>>", action_attr, page-1, tokens)
     else
-      id = assert(action.page, 'GoTo action must contain either an id or a page')
-      local tokens = action.tokens
-      if file then
-        action_attr = string.format("%s/S/GoToR/D[%i %s]>>", action_attr, id-1, tokens)
+      local page_objnum = pfile:reservepage(page)
+      action_attr = string.format("%s/S/GoTo/D[%i 0 R %s]>>", action_attr, page_objnum, tokens)
+    end
+  elseif action_type == 1 then -- GoTo
+    local id = assert(action.id, 'GoTo action must contain an id')
+    if file then
+      assert(type(id) == "string")
+      action_attr = action_attr .. "/S/GoToR/D" .. pdf_bytestring(id) .. ">>"
+    else
+      local dest = dests[id]
+      if not dest then
+        dest = pfile:getobj()
+        dests[id] = dest
+      end
+      if type(id) == "string" then
+        action_attr = action_attr .. "/S/GoTo/D" .. pdf_bytestring(id) .. ">>"
       else
-        local page_objnum = pfile:reservepage(id)
-        action_attr = string.format("%s/S/GoTo/D[%i 0 R %s]>>", action_attr, page_objnum, tokens)
+        action_attr = string.format("%s/S/GoTo/D %i 0 R>>", action_attr, dest)
       end
     end
   end
@@ -505,7 +548,7 @@ local colorstack_whatsit = declare_whatsit('pdf_colorstack', function(prop, p, n
   pdf.write(colorstack.mode, stack[#stack], x, y, p)
 end)
 local function write_colorstack()
-  local idx = token.scan_int()
+  local idx = scan_int()
   local colorstack = colorstacks[idx + 1]
   if not colorstack then
     tex.error('Undefined colorstack', {"The requested colorstack is not initialized. \z
@@ -513,10 +556,10 @@ local function write_colorstack()
         that you specified the wrong index. I will continue with colorstack 0."})
     colorstack = colorstacks[1]
   end
-  local action = token.scan_keyword'pop' and 'pop'
-              or token.scan_keyword'set' and 'set'
-              or token.scan_keyword'current' and 'current'
-              or token.scan_keyword'push' and 'push'
+  local action = scan_keyword'pop' and 'pop'
+              or scan_keyword'set' and 'set'
+              or scan_keyword'current' and 'current'
+              or scan_keyword'push' and 'push'
   if not action then
     tex.error('Missing action specifier for colorstack', {
         "I don't know what you want to do with this colorstack. I would have expected pop/set/current or push here. \z
@@ -525,8 +568,8 @@ local function write_colorstack()
   end
   local text
   if action == "push" or "set" then
-    text = token.scan_string()
-    -- text = token.to_string(token.scan_tokenlist()) -- Attention! This should never be executed in an expand-only context
+    text = scan_string()
+    -- text = token.serialize(token.scan_tokenlist()) -- Attention! This should never be executed in an expand-only context
   end
   local whatsit = node.new(whatsit_id, colorstack_whatsit)
   node.setproperty(whatsit, {
@@ -539,96 +582,103 @@ end
 local function scan_action()
   local action_type
   
-  if token.scan_keyword'user' then
-    return {action_type = 3, data = token.scan_string()}
-  elseif token.scan_keyword'thread' then
+  if scan_keyword'user' then
+    return {action_type = 3, data = scan_string()}
+  elseif scan_keyword'thread' then
     error[[FIXME: Unsupported]] -- TODO
-  elseif token.scan_keyword'goto' then
+  elseif scan_keyword'goto' then
     action_type = 1
   else
     error[[Unsupported action]]
   end
   local action = {
     action_type = action_type,
-    file = token.scan_keyword'file' and token.scan_string(),
+    file = scan_keyword'file' and scan_string(),
   }
-  if token.scan_keyword'page' then
+  if scan_keyword'page' then
     assert(action_type == 1)
-    local page = token.scan_int()
+    action_type = 0
+    action.action_type = 0
+    local page = scan_int()
     if page <= 0 then
-      error[[page must be positive in action specified]]
+      error[[page must be positive in action specification]]
     end
     action.page = page
-    action.tokens = token.scan_string()
-  elseif token.scan_keyword'num' then
+    action.tokens = scan_string()
+  elseif scan_keyword'num' then
     if action.file and action_type == 1 then
       error[[num style GoTo actions must be internal]]
     end
-    action.id = token.scan_int()
+    action.id = scan_int()
     if action.id <= 0 then
       error[[id must be positive]]
     end
-  elseif token.scan_keyword'name' then
-    action.id = token.scan_string()
+  elseif scan_keyword'name' then
+    action.id = scan_string()
   else
     error[[Unsupported id type]]
   end
-  action.new_window = token.scan_keyword'newwindow' and 1
-                   or token.scan_keyword'nonewwindow' and 2
+  action.new_window = scan_keyword'newwindow' and 1
+                   or scan_keyword'nonewwindow' and 2
   if action.new_window and not action.file then
     error[[newwindow is only supported for external files]]
   end
   return action
 end
 local function scan_literal_mode()
-  return token.scan_keyword"direct" and "direct"
-      or token.scan_keyword"page" and "page"
-      or token.scan_keyword"text" and "text"
-      or token.scan_keyword"direct" and "direct"
-      or token.scan_keyword"raw" and "raw"
+  return scan_keyword"direct" and "direct"
+      or scan_keyword"page" and "page"
+      or scan_keyword"text" and "text"
+      or scan_keyword"direct" and "direct"
+      or scan_keyword"raw" and "raw"
       or "origin"
 end
 local function maybe_gobble_cmd(cmd)
-  local t = token.scan_token()
+  local t = scan_token()
   if t.command ~= cmd then
     token.put_next(t)
   end
 end
 token.luacmd("pdffeedback", function()
-  if token.scan_keyword"colorstackinit" then
-    local page = token.scan_keyword'page'
-              or (token.scan_keyword'nopage' and false) -- If you want to pass "page" as mode
+  if scan_keyword"colorstackinit" then
+    local page = scan_keyword'page'
+              or (scan_keyword'nopage' and false) -- If you want to pass "page" as mode
     local mode = scan_literal_mode()
-    local default = token.scan_string()
+    local default = scan_string()
     tex.sprint(tostring(pdf.newcolorstack(default, mode, page)))
-  elseif token.scan_keyword"creationdate" then
+  elseif scan_keyword"creationdate" then
     tex.sprint(creationdate)
-  elseif token.scan_keyword"lastannot" then
+  elseif scan_keyword"lastannot" then
     tex.sprint(tostring(lastannot))
-  elseif token.scan_keyword"lastobj" then
+  elseif scan_keyword"lastobj" then
     tex.sprint(tostring(lastobj))
   else
   -- The following error message gobbles the next word as a side effect.
   -- This is intentional to make error-recovery easier.
-    error(string.format("Unknown PDF feedback %s", token.scan_word()))
+    error(string.format("Unknown PDF feedback %s", scan_word()))
   end
 end)
-token.luacmd("pdfextension", function(_, imm)
-  if token.scan_keyword"colorstack" then
+token.luacmd("pdfextension", function(_, immediate)
+  if immediate == "value" then return end
+  if immediate and immediate & 0x7 ~= 0 then
+    immediate = immediate & 0x8
+    tex.error("Unexpected prefix", "You used \\pdfextension with a prefix that doesn't belong there. I will ignore it for now.")
+  end
+  if scan_keyword"colorstack" then
     write_colorstack()
-  elseif token.scan_keyword"literal" then
+  elseif scan_keyword"literal" then
     local mode = scan_literal_mode()
-    local literal = token.scan_string()
+    local literal = scan_string()
     local whatsit = node.new(whatsit_id, literal_whatsit)
     node.setproperty(whatsit, {
         mode = mode,
         data = literal,
       })
     node.write(whatsit)
-  elseif token.scan_keyword"startlink" then
+  elseif scan_keyword"startlink" then
     local pfile = get_pfile()
     local whatsit = node.new(whatsit_id, start_link_whatsit)
-    local attr = token.scan_keyword'attr' and token.scan_string() or ''
+    local attr = scan_keyword'attr' and scan_string() or ''
     local action = scan_action()
     local objnum = pfile:getobj()
     lastannot = num
@@ -638,27 +688,27 @@ token.luacmd("pdfextension", function(_, imm)
         objnum = objnum,
       })
     node.write(whatsit)
-  elseif token.scan_keyword"endlink" then
+  elseif scan_keyword"endlink" then
     local whatsit = node.new(whatsit_id, end_link_whatsit)
     node.write(whatsit)
-  elseif token.scan_keyword"save" then
+  elseif scan_keyword"save" then
     local whatsit = node.new(whatsit_id, save_whatsit)
     node.write(whatsit)
-  elseif token.scan_keyword"setmatrix" then
-    local matrix = token.scan_string()
+  elseif scan_keyword"setmatrix" then
+    local matrix = scan_string()
     local whatsit = node.new(whatsit_id, setmatrix_whatsit)
     node.setproperty(whatsit, {
         data = matrix,
       })
     node.write(whatsit)
-  elseif token.scan_keyword"restore" then
+  elseif scan_keyword"restore" then
     local whatsit = node.new(whatsit_id, restore_whatsit)
     node.write(whatsit)
-  elseif token.scan_keyword"info" then
-    infodir = infodir .. token.scan_string()
-  elseif token.scan_keyword"catalog" then
-    catalogdir = catalogdir .. ' ' .. token.scan_string()
-    if token.scan_keyword'openaction' then
+  elseif scan_keyword"info" then
+    infodir = infodir .. scan_string()
+  elseif scan_keyword"catalog" then
+    catalogdir = catalogdir .. ' ' .. scan_string()
+    if scan_keyword'openaction' then
       if catalog_openaction then
         tex.error("Duplicate openaction", {"Only one use of \\pdfextension catalog is allowed to \z
             have an openaction."})
@@ -667,19 +717,19 @@ token.luacmd("pdfextension", function(_, imm)
         catalog_openaction = get_action_attr(get_pfile(), action)
       end
     end
-  elseif token.scan_keyword"names" then
-    namesdir = namesdir .. ' ' .. token.scan_string()
-  elseif token.scan_keyword"obj" then
+  elseif scan_keyword"names" then
+    namesdir = namesdir .. ' ' .. scan_string()
+  elseif scan_keyword"obj" then
     local pfile = get_pfile()
-    if token.scan_keyword"reserveobjnum" then
+    if scan_keyword"reserveobjnum" then
       lastobj = pfile:getobj()
     else
-      local num = token.scan_keyword'useobjnum' and token.scan_int() or pfile:getobj()
+      local num = scan_keyword'useobjnum' and scan_int() or pfile:getobj()
       lastobj = num
-      local attr = token.scan_keyword'stream' and (token.scan_keyword'attr' and token.scan_string() or '')
-      local isfile = token.scan_keyword'file'
-      local content = token.scan_string()
-      if immediate then
+      local attr = scan_keyword'stream' and (scan_keyword'attr' and scan_string() or '')
+      local isfile = scan_keyword'file'
+      local content = scan_string()
+      if immediate == 8 then
         if attr then
           pfile:stream(num, attr, content, isfile)
         else
@@ -693,43 +743,43 @@ token.luacmd("pdfextension", function(_, imm)
         end
       end
     end
-  elseif token.scan_keyword"refobj" then
-    local num = token.scan_int()
+  elseif scan_keyword"refobj" then
+    local num = scan_int()
     local whatsit = node.new(whatsit_id, refobj_whatsit)
     node.setproperty(whatsit, {
         obj = num,
       })
     node.write(whatsit)
-  elseif token.scan_keyword"outline" then
+  elseif scan_keyword"outline" then
     local pfile = get_pfile()
-    local attr = token.scan_keyword'attr' and token.scan_string() or ''
+    local attr = scan_keyword'attr' and scan_string() or ''
     local action
-    if token.scan_keyword"useobjnum" then
-      action = token.scan_int()
+    if scan_keyword"useobjnum" then
+      action = scan_int()
     else
       local actionobj = scan_action()
       action = pfile:indirect(nil, get_action_attr(pfile, actionobj))
     end
     local outline = get_outline()
-    if token.scan_keyword'level' then
-      local level = token.scan_int()
-      local open = token.scan_keyword'open'
-      local title = token.scan_string()
+    if scan_keyword'level' then
+      local level = scan_int()
+      local open = scan_keyword'open'
+      local title = scan_string()
       outline:add(pdf_text(title), action, level, open, attr)
     else
-      local count = token.scan_keyword'count' and token.scan_int() or 0
-      local title = token.scan_string()
+      local count = scan_keyword'count' and scan_int() or 0
+      local title = scan_string()
       outline:add_legacy(pdf_text(title), action, count, attr)
     end
-  elseif token.scan_keyword"dest" then
+  elseif scan_keyword"dest" then
     local id
-    if token.scan_keyword'num' then
-      id = token.scan_int()
+    if scan_keyword'num' then
+      id = scan_int()
       if id <= 0 then
         error[[id must be positive]]
       end
-    elseif token.scan_keyword'name' then
-      id = token.scan_string()
+    elseif scan_keyword'name' then
+      id = scan_string()
     else
       error[[Unsupported id type]]
     end
@@ -738,54 +788,54 @@ token.luacmd("pdfextension", function(_, imm)
       dest_id = id,
     }
     node.setproperty(whatsit, prop)
-    if token.scan_keyword'xyz' then
+    if scan_keyword'xyz' then
       prop.dest_type = 'xyz'
-      prop.xyz_zoom = token.scan_keyword'zoom' and token.scan_int()
+      prop.xyz_zoom = scan_keyword'zoom' and scan_int()
       maybe_gobble_cmd(spacer_cmd)
-    elseif token.scan_keyword'fitr' then
+    elseif scan_keyword'fitr' then
       prop.dest_type = 'fitr'
       maybe_gobble_cmd(spacer_cmd)
       while true do
-        if token.scan_keyword'width' then
-          prop.width = token.scan_dimen()
-        elseif token.scan_keyword'height' then
-          prop.height = token.scan_dimen()
-        elseif token.scan_keyword'depth' then
-          prop.depth = token.scan_dimen()
+        if scan_keyword'width' then
+          prop.width = scan_dimen()
+        elseif scan_keyword'height' then
+          prop.height = scan_dimen()
+        elseif scan_keyword'depth' then
+          prop.depth = scan_dimen()
         else
           break
         end
       end
-    elseif token.scan_keyword'fitbh' then
+    elseif scan_keyword'fitbh' then
       prop.dest_type = 'fitbh'
       maybe_gobble_cmd(spacer_cmd)
-    elseif token.scan_keyword'fitbv' then
+    elseif scan_keyword'fitbv' then
       prop.dest_type = 'fitbv'
       maybe_gobble_cmd(spacer_cmd)
-    elseif token.scan_keyword'fitb' then
+    elseif scan_keyword'fitb' then
       prop.dest_type = 'fitb'
       maybe_gobble_cmd(spacer_cmd)
-    elseif token.scan_keyword'fith' then
+    elseif scan_keyword'fith' then
       prop.dest_type = 'fith'
       maybe_gobble_cmd(spacer_cmd)
-    elseif token.scan_keyword'fitv' then
+    elseif scan_keyword'fitv' then
       prop.dest_type = 'fitv'
       maybe_gobble_cmd(spacer_cmd)
-    elseif token.scan_keyword'fit' then
+    elseif scan_keyword'fit' then
       prop.dest_type = 'fit'
       maybe_gobble_cmd(spacer_cmd)
     else
       error[[Unsupported dest type]]
     end
     node.write(whatsit)
-  elseif token.scan_keyword'mapline' then
-    fontmap.mapline(token.scan_string())
+  elseif scan_keyword'mapline' then
+    fontmap.mapline(scan_string())
   else
   -- The following error message gobbles the next word as a side effect.
   -- This is intentional to make error-recovery easier.
-    error(string.format("Unknown PDF extension %s", token.scan_word()))
+    error(string.format("Unknown PDF extension %s", scan_word()))
   end
-end, "protected")
+end, "value")
 local imglib = require'luametalatex-pdf-image'
 local imglib_node = imglib.node
 local imglib_write = imglib.write
@@ -802,19 +852,25 @@ local lastimage = -1
 local lastimagepages = -1
 
 -- These are very minimal right now but LaTeX isn't using the scaling etc. stuff anyway.
-token.luacmd("saveimageresource", function(imm)
-  local attr = token.scan_keyword'attr' and token.scan_string() or nil
-  local page = token.scan_keyword'page' and token.scan_int() or nil
-  local userpassword = token.scan_keyword'userpassword' and token.scan_string() or nil
-  local ownerpassword = token.scan_keyword'ownerpassword' and token.scan_string() or nil
-  -- local colorspace = token.scan_keyword'colorspace' and token.scan_int() or nil -- Doesn't make sense for PDF
-  local pagebox = token.scan_keyword'mediabox' and 'media'
-               or token.scan_keyword'cropbox' and 'crop'
-               or token.scan_keyword'bleedbox' and 'bleed'
-               or token.scan_keyword'trimbox' and 'trim'
-               or token.scan_keyword'artbox' and 'art'
+token.luacmd("saveimageresource", function(_, immediate)
+  if immediate == "value" then return end
+  if immediate and immediate & 0x7 ~= 0 then
+    print(immediate)
+    immediate = immediate & 0x8
+    tex.error("Unexpected prefix", "You used \\saveimageresource with a prefix that doesn't belong there. I will ignore it for now.")
+  end
+  local attr = scan_keyword'attr' and scan_string() or nil
+  local page = scan_keyword'page' and scan_int() or nil
+  local userpassword = scan_keyword'userpassword' and scan_string() or nil
+  local ownerpassword = scan_keyword'ownerpassword' and scan_string() or nil
+  -- local colorspace = scan_keyword'colorspace' and scan_int() or nil -- Doesn't make sense for PDF
+  local pagebox = scan_keyword'mediabox' and 'media'
+               or scan_keyword'cropbox' and 'crop'
+               or scan_keyword'bleedbox' and 'bleed'
+               or scan_keyword'trimbox' and 'trim'
+               or scan_keyword'artbox' and 'art'
                or nil
-  local filename = token.scan_string()
+  local filename = scan_string()
   local img = imglib.scan{
     attr = attr,
     page = page,
@@ -826,30 +882,26 @@ token.luacmd("saveimageresource", function(imm)
   local pfile = get_pfile()
   lastimage = imglib.get_num(pfile, img)
   lastimagepages = img.pages or 1
-  if imm == 'immediate' then
+  if immediate == 8 then
     imglib_immediatewrite(pfile, img)
   end
-end, "protected")
+end, "value")
 
 token.luacmd("useimageresource", function()
   local pfile = get_pfile()
-  local img = assert(imglib.from_num(token.scan_int()))
+  local img = assert(imglib.from_num(scan_int()))
   imglib_write(pfile, img)
 end, "protected")
 
-local value_values = token.values'value'
-for i=0,#value_values do
-  value_values[value_values[i]] = i
-end
-local integer_code = value_values.integer
+local integer_code = token.value.integer
 
 token.luacmd("lastsavedimageresourceindex", function()
   return integer_code, lastimage
-end, "protected", "value")
+end, "value")
 
 token.luacmd("lastsavedimageresourcepages", function()
   return integer_code, lastimagepages
-end, "protected", "value")
+end, "value")
 
 local savedbox = require'luametalatex-pdf-savedbox'
 local savedbox_save = savedbox.save
@@ -860,47 +912,78 @@ function tex.saveboxresource(n, attr, resources, immediate, type, margin, pfile)
       error[[Invalid argument to saveboxresource]]
     end
     token.put_next(token.create'box', token.new(n, token.command_id'char_given'))
-    n = token.scan_list()
+    n = scan_box()
   end
-  margin = margin or tex.sp'1bp' -- FIXME: default margin variable
+  margin = margin or pdfvariable.xformmargin
   return savedbox_save(pfile or get_pfile(), n, attr, resources, immediate, type, margin, fontdirs, usedglyphs)
 end
 tex.useboxresource = savedbox.use
 
 local lastbox = -1
 
-token.luacmd("saveboxresource", function(imm)
-  local type
-  if token.scan_keyword'type' then
-    texio.write_nl('XForm type attribute ignored')
-    type = token.scan_int()
+token.luacmd("saveboxresource", function(_, immediate)
+  if immediate == "value" then return end
+  if immediate and immediate & 0x7 ~= 0 then
+    immediate = immediate & 0x8
+    tex.error("Unexpected prefix", "You used \\saveboxresource with a prefix that doesn't belong there. I will ignore it for now.")
   end
-  local attr = token.scan_keyword'attr' and token.scan_string() or nil
-  local resources = token.scan_keyword'resources' and token.scan_string() or nil
-  local margin = token.scan_keyword'margin' and token.scan_dimen() or nil
-  local box = token.scan_int()
+  local type
+  if scan_keyword'type' then
+    texio.write_nl('XForm type attribute ignored')
+    type = scan_int()
+  end
+  local attr = scan_keyword'attr' and scan_string() or nil
+  local resources = scan_keyword'resources' and scan_string() or nil
+  local margin = scan_keyword'margin' and scan_dimen() or nil
+  local box = scan_int()
 
-  local index = tex.saveboxresource(box, attr, resources, imm == 'immediate', type, margin)
+  local index = tex.saveboxresource(box, attr, resources, immediate == 8, type, margin)
   lastbox = index
-end, "protected")
+end, "value")
 
 token.luacmd("useboxresource", function()
   local width, height, depth
   while true do
-    if token.scan_keyword'width' then
-      width = token.scan_dimen()
-    elseif token.scan_keyword'height' then
-      height = token.scan_dimen()
-    elseif token.scan_keyword'depth' then
-      depth = token.scan_dimen()
+    if scan_keyword'width' then
+      width = scan_dimen()
+    elseif scan_keyword'height' then
+      height = scan_dimen()
+    elseif scan_keyword'depth' then
+      depth = scan_dimen()
     else
       break
     end
   end
-  local index = token.scan_int()
+  local index = scan_int()
   node.write((tex.useboxresource(index, width, height, depth)))
 end, "protected")
 
 token.luacmd("lastsavedboxresourceindex", function()
   return integer_code, lastbox
-end, "protected", "value")
+end, "value")
+
+local saved_pos_x, saved_pos_y = -1, -1
+local save_pos_whatsit = declare_whatsit('save_pos', function(_, _, _, x, y)
+  saved_pos_x, saved_pos_y = assert(math.tointeger(x)), assert(math.tointeger(y))
+end)
+token.luacmd("savepos", function() -- \savepos
+  return node.direct.write(node.direct.new(whatsit_id, save_pos_whatsit))
+end, "protected")
+
+token.luacmd("lastxpos", function()
+  return integer_code, (saved_pos_x+.5)//1
+end, "value")
+
+token.luacmd("lastypos", function()
+  return integer_code, (saved_pos_y+.5)//1
+end, "value")
+
+local function pdf_register_funcs(name)
+  pdf[name] = ""
+  pdf['get' .. name] = function() return pdf[name] end
+  pdf['set' .. name] = function(s) pdf[name] = assert(s) end
+end
+
+pdf_register_funcs'pageattributes'
+pdf_register_funcs'pageresources'
+pdf_register_funcs'pagesattributes'

luametalatex-basecallbacks.lua

@@ -0,0 +1,144 @@
+-- Three callbacks are defined in other files: stop_run in back-pdf, pre_dump in lateinit, and find_fmt_file in init
+
+local read_tfm = font.read_tfm
+local font_define = font.define
+local callbacks = require'luametalatex-callbacks'
+
+if status.ini_version then
+  function callbacks.define_font(name, size)
+    local f = read_tfm(name, size)
+    if not f then return end
+    local id = font_define(f)
+    lua.prepared_code[#lua.prepared_code+1] = string.format("assert(%i == font.define(font.read_tfm(%q, %i)))", id, name, size)
+    return id
+  end
+else
+  function callbacks.define_font(name, size)
+    local f = read_tfm(name, size)
+    if not f then
+      tex.error(string.format("Font %q not found", name), "The requested font could't be loaded.\n\z
+        Are you sure that you passed the right name and\n\z
+        that the font is actually installed?")
+      return 0
+    end
+    return font.define(f)
+  end
+end
+callbacks.__freeze'define_font'
+
+function callbacks.find_log_file(name) return name end
+callbacks.__freeze'find_log_file'
+
+-- find_data_file is not an engine callback in luametatex, so we don't __freeze it
+if status.ini_version then
+  function unhook_expl()
+    callbacks.find_data_file = nil
+  end
+  function callbacks.find_data_file(name)
+    if name == 'ltexpl.ltx' then
+      name = 'luametalatex-ltexpl-hook'
+    end
+    return kpse.find_file(name, 'tex', true)
+  end
+end
+  local function normal_find_data_file(name)
+    return kpse.find_file(name, 'tex', true)
+  end
+function callbacks.open_data_file(name)
+  local find_callback = callbacks.find_data_file
+  local path
+  if find_callback then
+    path = find_callback(name)
+  else
+    path = kpse.find_file(name, 'tex', true)
+  end
+  if not path then return end
+
+  local open_callback = callbacks.open_data_file
+  if open_callback then
+    return open_callback(path)
+  end
+
+  local f = io.open(path, 'r')
+  return f and setmetatable({
+    reader = function()
+      local line = f:read()
+      return line
+    end,
+    close = function() f:close() f = nil end,
+  }, {
+    __gc = function() if f then f:close() end end,
+  })
+end
+callbacks.__freeze('open_data_file', true)
+
+local do_terminal_input do
+  local function terminal_open_data_file()
+    local old = callbacks.open_data_file
+    return function()
+      callbacks.open_data_file = old
+      return {
+        reader = function()
+          texio.write_nl('term', '* ')
+          local line = io.stdin:read()
+          return line
+        end,
+        close = function() end,
+      }
+    end
+  end
+  function do_terminal_input()
+    local old_find = callbacks.find_data_file
+    function callbacks.find_data_file(name) 
+      callbacks.find_data_file = old_find 
+      return name
+    end
+    callbacks.open_data_file = terminal_open_data_file()
+    token.put_next(token.create'expandafter', token.create'relax', token.create'input', 'TERMINAL ')
+    token.skip_next_expanded()
+  end
+end
+
+local errorvalues = tex.geterrorvalues()
+function callbacks.intercept_tex_error(mode, errortype)
+  errortype = errorvalues[errortype]
+  if errortype == "eof" then
+    tex.runlocal(function()token.put_next(token.create'tracingall')end)
+    do_terminal_input()
+    tex.runlocal(token.skip_next)
+    return 3
+  end
+  texio.write'.'
+  tex.showcontext()
+  if mode ~= 3 then return mode end
+  repeat
+    texio.write_nl'? '
+    local line = io.read()
+    if not line then
+      tex.fatalerror'End of line encountered on terminal'
+    end
+    if line == "" then return 3 end
+    local first = line:sub(1,1):upper()
+    if first == 'H' then
+      texio.write(tex.gethelptext() or "Sorry, I don't know how to help in this situation.\n\z
+        Maybe you should try asking a human?")
+    elseif first == 'I' then
+      line = line:sub(2)
+      tex.runlocal(function()
+        tex.sprint(token.scan_token(), line)
+      end)
+      return 3
+    elseif first == 'Q' then texio.write'OK, entering \\batchmode...\n' return 0
+    elseif first == 'R' then texio.write'OK, entering \\nonstopmode...\n' return 1
+    elseif first == 'S' then texio.write'OK, entering \\scrollmode...\n' return 2
+    elseif first == 'X' then return -1
+    else
+      texio.write'Type <return> to proceed, S to scroll future error messages,\
+\z      R to run without stopping, Q to run quietly,\
+\z      I to insert something,\
+\z      H for help, X to quit.'
+    end
+  until false
+  return 3
+end
+callbacks.__freeze'intercept_tex_error'

luametalatex-baseregisters.lua

@@ -1,9 +1,9 @@
-local value_values = token.values'value'
-for i=0,#value_values do
-  value_values[value_values[i]] = i
-end
-local count_code = value_values.integer
-local dimen_code = value_values.dimension
+local scan_dimen = token.scan_dimen
+local scan_int = token.scan_integer
+local scan_keyword = token.scan_keyword
+
+local count_code = token.value.integer
+local dimen_code = token.value.dimension
 
 local set_local = require'luametalatex-local'
 
@@ -30,10 +30,10 @@ local function tex_variable(value, scanner, name, default)
     if scanning == 'value' then
       return value, tex_variables[name]
     else
-      token.scan_keyword'='
-      return set_local(tex_variables, name, scanner(), scanning == 'global')
+      scan_keyword'='
+      return set_local(tex_variables, name, scanner(), scanning and scanning & 4 == 4)
     end
-  end, 'global', 'protected', 'value')
+  end, 'global', 'value')
   if status.ini_version then
     tex_variables[name] = default
   end
@@ -99,14 +99,14 @@ local function pdf_variable(value, scanner, name, default, force_default)
     if scanning == 'value' then
       return value, real_pdf_variables[name]
     elseif force_default then
-      token.scan_keyword'='
+      scan_keyword'='
       local new = scanner()
       if new ~= default then
         texio.write_nl('term and log', string.format("Unsupported PDF variable: \z
             %q is not supported and fixed to %i, but you tried to set it to %i", name, default, new))
       end
     else
-      token.scan_keyword'='
+      scan_keyword'='
       return set_local(real_pdf_variables, name, scanner(), scanning == 'global')
     end
   end, 'global', 'protected', 'value')
@@ -115,42 +115,43 @@ local function pdf_variable(value, scanner, name, default, force_default)
   end
 end
 
-tex_variable(count_code, token.scan_int, 'suppressfontnotfounderror', 0)
-tex_variable(count_code, token.scan_int, 'outputmode', 1) -- The "traditional" default would be 0,
-                                                            -- but we do not actually support that.
-tex_variable(dimen_code, token.scan_dimen, 'pageheight', tex.sp'297mm')
-tex_variable(dimen_code, token.scan_dimen, 'pagewidth', tex.sp'210mm')
+tex_variable(count_code, scan_int, 'suppressfontnotfounderror', 0)
+tex_variable(count_code, scan_int, 'outputmode', 1) -- The "traditional" default would be 0,
+                                                          -- but we do not actually support that.
+tex_variable(dimen_code, scan_dimen, 'pageheight', 0)
+tex_variable(dimen_code, scan_dimen, 'pagewidth', 0)
 
-tex_variable(count_code, token.scan_int, 'bodydirection', 0)
-tex_variable(count_code, token.scan_int, 'pagedirection', 0)
+tex_variable(count_code, scan_int, 'bodydirection', 0)
+tex_variable(count_code, scan_int, 'pagedirection', 0)
 
-pdf_variable(dimen_code, token.scan_dimen, 'horigin', tex.sp'1in')
-pdf_variable(dimen_code, token.scan_dimen, 'vorigin', tex.sp'1in')
-pdf_variable(dimen_code, token.scan_dimen, 'linkmargin', tex.sp'0pt')
-pdf_variable(dimen_code, token.scan_dimen, 'destmargin', tex.sp'0pt')
-pdf_variable(dimen_code, token.scan_dimen, 'threadmargin', tex.sp'0pt') -- We don't support threads, so this isn't doing anything
-pdf_variable(count_code, token.scan_int, 'majorversion', 1)
-pdf_variable(count_code, token.scan_int, 'minorversion', 7)
-pdf_variable(count_code, token.scan_int, 'compresslevel', 9)
-pdf_variable(count_code, token.scan_int, 'objcompresslevel', 3)
+pdf_variable(dimen_code, scan_dimen, 'horigin', tex.sp'1in')
+pdf_variable(dimen_code, scan_dimen, 'vorigin', tex.sp'1in')
+pdf_variable(dimen_code, scan_dimen, 'linkmargin', tex.sp'0pt')
+pdf_variable(dimen_code, scan_dimen, 'destmargin', tex.sp'0pt')
+pdf_variable(dimen_code, scan_dimen, 'xformmargin', tex.sp'0pt')
+pdf_variable(dimen_code, scan_dimen, 'threadmargin', tex.sp'0pt', true) -- We don't support threads, so this isn't doing anything
+pdf_variable(count_code, scan_int, 'majorversion', 1)
+pdf_variable(count_code, scan_int, 'minorversion', 7)
+pdf_variable(count_code, scan_int, 'compresslevel', 9)
+pdf_variable(count_code, scan_int, 'objcompresslevel', 3)
 
-pdf_variable(count_code, token.scan_int, 'decimaldigits', 4, true) -- Will probably stay fixed, but should be more consistent
-pdf_variable(count_code, token.scan_int, 'gentounicode', 0, true) -- We expect the fontloader to generade tounicode tables. Might change at some point
+pdf_variable(count_code, scan_int, 'decimaldigits', 4, true) -- Will probably stay fixed, but should be more consistent
+pdf_variable(count_code, scan_int, 'gentounicode', 0, true) -- We expect the fontloader to generade tounicode tables. Might change at some point
 -- These two are ignored, but that is consistent with pdfTeX as long as imageapplygamma is 0:
-pdf_variable(count_code, token.scan_int, 'gamma', 1000)
-pdf_variable(count_code, token.scan_int, 'imagegamma', 1000)
-pdf_variable(count_code, token.scan_int, 'imageapplygamma', 0, true)
-pdf_variable(count_code, token.scan_int, 'imagehicolor', 1, true) -- We don't consider ancient PDF versions, no no reason to strip images
-pdf_variable(count_code, token.scan_int, 'imageaddfilename', 0, true) -- Could be added, but I never saw a reason for this anyway.
-pdf_variable(count_code, token.scan_int, 'inclusionerrorlevel', -1, true) -- FIXME: At least a warning should be supported
-pdf_variable(count_code, token.scan_int, 'inclusioncopyfonts', 0, true) -- Would be fragile and restrict our ability to use "creative" font constructs
-pdf_variable(count_code, token.scan_int, 'uniqueresname', 0, true) -- I add this if you show me a usecase
-pdf_variable(count_code, token.scan_int, 'pagebox', 2, true) -- TODO (1: media, 2: crop, 3: bleed, 4: trim, 5: art
-pdf_variable(count_code, token.scan_int, 'forcepagebox', 0, true) -- Considered obsolete even in pdfTeX
-pdf_variable(count_code, token.scan_int, 'imageresolution', 72, true) -- TODO Also 0 should be the same as 72 ?!?!?!?
+pdf_variable(count_code, scan_int, 'gamma', 1000)
+pdf_variable(count_code, scan_int, 'imagegamma', 1000)
+pdf_variable(count_code, scan_int, 'imageapplygamma', 0, true)
+pdf_variable(count_code, scan_int, 'imagehicolor', 1, true) -- We don't consider ancient PDF versions, no no reason to strip images
+pdf_variable(count_code, scan_int, 'imageaddfilename', 0, true) -- Could be added, but I never saw a reason for this anyway.
+pdf_variable(count_code, scan_int, 'inclusionerrorlevel', -1, true) -- FIXME: At least a warning should be supported
+pdf_variable(count_code, scan_int, 'inclusioncopyfonts', 0, true) -- Would be fragile and restrict our ability to use "creative" font constructs
+pdf_variable(count_code, scan_int, 'uniqueresname', 0, true) -- I add this if you show me a usecase
+pdf_variable(count_code, scan_int, 'pagebox', 2, true) -- TODO (1: media, 2: crop, 3: bleed, 4: trim, 5: art
+pdf_variable(count_code, scan_int, 'forcepagebox', 0, true) -- Considered obsolete even in pdfTeX
+pdf_variable(count_code, scan_int, 'imageresolution', 72, true) -- TODO Also 0 should be the same as 72 ?!?!?!?
 
-pdf_variable(count_code, token.scan_int, 'pkresolution', 1200) -- Original default is 72, but that's crazy
-pdf_variable(count_code, token.scan_int, 'pkfixeddpi', 0) -- TODO: Implemented, but even when set to one, font sharing doesn't adapt yet.
+pdf_variable(count_code, scan_int, 'pkresolution', 1200) -- Original default is 72, but that's crazy
+pdf_variable(count_code, scan_int, 'pkfixeddpi', 0) -- TODO: Implemented, but even when set to one, font sharing doesn't adapt yet.
                                                           -- Changing that is complicated because it has to be known pretty early.
 pdf_toks('pkmode', '')
 
@@ -174,6 +175,9 @@ local dir_regs = require 'luametalatex-dir-registers'
 dir_regs 'textdir'
 dir_regs 'bodydir'
 dir_regs 'pagedir'
+dir_regs 'pardir'
+dir_regs 'linedir'
+dir_regs 'mathdir'
 
 if status.ini_version then
   -- Run in pre_dump callback:

luametalatex-build-bytecode.lua

@@ -14,7 +14,7 @@ first = first .. later
 
 local list = {}
 
-return function(t)
+return true and function(t) return '' end or function(t)
   local length = #t
   local tmpl = first
   for i, mod in ipairs(t) do

luametalatex-callbacks.lua

@@ -1,84 +1,69 @@
--- Two callbacks are defined in other files: pre_dump in lateinit and find_fmt_file in init
+-- Now overwrite the callback functionality. Our system is based on the ssumption there there are
+-- no unknown callback names, just callbacks very unlikely to ever be called. That doesn't lead to
+-- good error checking, but we expect this to be overwritten by LaTeX anyway.
+--
+-- There are four callback types on this level:
+--  1. luametalatex defined callbacks. They are not real engine callbacks, the code using them is
+--     responsible for their potential non-existance.
+--  2. Engine callbacks not defined by us. They are simply passed on to the engine. All engine
+--     callbacks are set to this by default.
+--  3. Engine callbacks with a provided default. There is a luametalatex implementation, but it
+--     can be overwritten by the user. If the user disabled their implementation, so provided
+--     default is restored.
+--  4. Engine callbacks with mandatory code. The luametalatex implementation can not be overwitten
+--     by the user, but a luametalatex-defined callback is added with the same name.
+--
+--  A callback has type 1 or type 4 if is_user_callback is true. If it has type 4, is_user_callback
+--  has to be set manually and in addition, an implementation if the system callback is registered.
+--
+--  A callback has type 3, if is_user_callback is false and system_callbacks is defined.
 
-local read_tfm = font.read_tfm
-local font_define = font.define
+local callback_known = callback.known
+local callback_find = callback.find
 local callback_register = callback.register
+local rawset = rawset
+local system_callbacks = {}
+local is_user_callback = setmetatable({}, {
+  __index = function(t, name)
+    local is_user = not callback_known(name)
+    t[name] = is_user
+    return is_user
+  end,
+})
+local callbacks = setmetatable({
+  __freeze = function(name, fixed)
+    -- Convert from type 2 to type 3 or 4. This function will be deleted before user code runs.
+    assert(not is_user_callback[name], 'Not a system callback')
+    assert(not system_callbacks[name], 'Already frozen')
+    is_user_callback[name] = fixed and true or false
+    system_callbacks[name] = callback_find(name)
+    assert(system_callbacks[name], 'Attempt to freeze undefined callback')
+  end,
+}, {
+  __index = function(cbs, name)
+    if is_user_callback[name] then
+      -- Avoid repetitive lookups
+      rawset(cbs, name, false)
+      return false
+    end
+    return callback_find(name)
+  end,
+  __newindex = function(cbs, name, new)
+    if is_user_callback[name] then
+      -- Avoid repetitive lookups
+      rawset(cbs, name, new or false)
+      return
+    end
+    return callback_register(name, new or system_callbacks[name])
+  end,
+})
 
-if status.ini_version then
-  callback_register('define_font', function(name, size)
-    local f = read_tfm(name, size)
-    if not f then return end
-    local id = font_define(f)
-    lua.prepared_code[#lua.prepared_code+1] = string.format("assert(%i == font.define(font.read_tfm(%q, %i)))", id, name, size)
-    return id
-  end)
-else
-  callback_register('define_font', function(name, size)
-    local f = read_tfm(name, size)
-    if not f then return end
-    return font.define(f)
-  end)
+function callback.register(name, new)
+  callbacks[name] = new
 end
-callback_register('find_log_file', function(name) return name end)
-do
-  local function normal_find_data_file(name)
-    return kpse.find_file(name, 'tex', true)
-  end
-  if status.ini_version then
-    function unhook_expl()
-      callback_register('find_data_file', normal_find_data_file)
-    end
-    callback_register('find_data_file', function(name)
-      if name == 'ltexpl.ltx' then
-        name = 'luametalatex-ltexpl-hook'
-      end
-      return normal_find_data_file(name)
-    end)
-  else
-    callback_register('find_data_file', normal_find_data_file)
-  end
+-- The and ... or construction makes sure that even in raw mode, non-engine callbacks are found
+function callback.find(name, raw)
+  return raw and callback_find(name) or callbacks[name]
 end
--- callback_register('read_data_file', function(name) error[[TODO]]return kpse.find_file(name, 'tex', true) end)
-callback_register('open_data_file', function(name)
-  local f = io.open(name)
-  return setmetatable({
-    reader = function()
-      local line = f:read()
-      return line
-    end,
-    close = function()error[[1]] return f:close() end,
-  }, {
-    __gc = function()f:close()end,
-  })
-end)
-callback_register('handle_error_hook', function()
-  repeat
-    texio.write_nl'? '
-    local line = io.read()
-    if not line then
-      tex.fatalerror'End of line encountered on terminal'
-    end
-    if line == "" then return 3 end
-    local first = line:sub(1,1):upper()
-    if first == 'H' then
-      texio.write(tex.gethelptext() or "Sorry, I don't know how to help in this situation.\n\z
-        Maybe you should try asking a human?")
-    elseif first == 'I' then
-      line = line:sub(2)
-      tex.runtoks(function()
-        tex.sprint(token.scan_token(), line)
-      end)
-      return 3
-    elseif first == 'Q' then texio.write'OK, entering \\batchmode...\n' return 0
-    elseif first == 'R' then texio.write'OK, entering \\nonstopmode...\n' return 1
-    elseif first == 'S' then texio.write'OK, entering \\scrollmode...\n' return 2
-    elseif first == 'X' then return -1
-    else
-      texio.write'Type <return> to proceed, S to scroll future error messages,\
-\z      R to run without stopping, Q to run quietly,\
-\z      I to insert something,\
-\z      H for help, X to quit.'
-    end
-  until false
-  return 3
-end)
+
+return callbacks

luametalatex-dir-registers.lua

@@ -1,32 +1,30 @@
+local scan_int = token.scan_integer
+local scan_keyword = token.scan_keyword
+
 -- local names = {}
 local setters = {
 }
 local getters = {
 }
-local value_values = token.values'value'
-for i=0,#value_values do
-  value_values[value_values[i]] = i
-end
 function tex.gettextdir() return tex.textdirection end
 function tex.getlinedir() return tex.linedirection end
 function tex.getmathdir() return tex.mathdirection end
 function tex.getpardir()  return tex.pardirection  end
-local integer_code = value_values.integer
+local integer_code = token.value.integer
 local function set_xdir(id, scanning)
   if scanning == 'value' then
-    print(scanning)
     return integer_code, getters[id]()
   end
   -- local global = scanning == 'global'
   local value
-  if token.scan_keyword'tlt' then
+  if scan_keyword'tlt' then
     value = 0
-  elseif token.scan_keyword'trt' then
+  elseif scan_keyword'trt' then
     value = 1
   else
-    value = token.scan_int()
+    value = scan_int()
   end
-  setters[id](value)
+  setters[id](value, scanning)
 end
 return function(name)
   local getter = tex["get" .. name]

luametalatex-firstcode.lua

@@ -1,3 +1,15 @@
+local scan_int = token.scan_integer
+token.scan_int = scan_int -- For compatibility with LuaTeX packages
+local scan_token = token.scan_token
+local scan_tokenlist = token.scan_tokenlist
+local scan_keyword = token.scan_keyword
+local scan_csname = token.scan_csname
+local set_macro = token.set_macro
+
+local callback_find = callback.find
+
+local constants = status.getconstants()
+
 local lua_call_cmd = token.command_id'lua_call'
 local properties = node.direct.get_properties_table()
 node.direct.properties = properties
@@ -20,10 +32,10 @@ local function scan_filename()
   local quoted = false
   local tok, cmd
   repeat
-    tok = token.scan_token()
+    tok = scan_token()
     cmd = tok.command
   until cmd ~= spacer_cmd and cmd ~= relax_cmd
-  while (tok.command <= 12 and tok.command > 0 and tok.command ~= 9 and tok.index <= token.biggest_char()
+  while (tok.command <= 12 and tok.command > 0 and tok.command ~= 9 and tok.index <= constants.max_character_code
           or (token.put_next(tok) and false))
       and (quoted or tok.index ~= string.byte' ') do
     if tok.index == string.byte'"' then
@@ -31,39 +43,68 @@ local function scan_filename()
     else
       name[#name+1] = tok.index
     end
-    tok = token.scan_token()
+    tok = scan_token()
   end
   return utf8.char(table.unpack(name))
 end
 
+-- These are chosen to coincide with ltluatex's default catcodetables.
+-- In expl-hook, we check that the values are as expected.
+local initex_catcodetable = 1
+local string_catcodetable = 2
+if status.ini_version then
+  tex.runlocal(function()tex.sprint[[\initcatcodetable 1\initcatcodetable 2]]end)
+  local setcatcode = tex.setcatcode
+  for i=0,127 do
+    setcatcode('global', 2, i, 12)
+  end
+  setcatcode('global', 2, 32, 10)
+end
+
 local l = lpeg or require'lpeg'
 local add_file_extension = l.Cs((1-('.' * (1-l.S'./\\')^0) * -1)^0 * (l.P(1)^1+l.Cc'.tex'))
-local ofiles = {}
+local ofiles, ifiles = {}, {}
 local function do_openout(p)
   if ofiles[p.file] then
-    error[[Existing file]]
-  else
-    local msg
-    ofiles[p.file], msg = io.open(add_file_extension:match(p.name), 'w')
-    if not ofiles[p.file] then
-      error(msg)
-    end
+    ofiles[p.file]:close()
+  end
+  local msg
+  ofiles[p.file], msg = io.open(add_file_extension:match(p.name), 'w')
+  if not ofiles[p.file] then
+    error(msg)
   end
 end
 local open_whatsit = new_whatsit('open', do_openout)
 token.luacmd("openout", function(_, immediate) -- \openout
-  local file = token.scan_int()
-  token.scan_keyword'='
+  if immediate == "value" then return end
+  if immediate and immediate & 0x7 ~= 0 then
+    immediate = immediate & 0x8
+    tex.error("Unexpected prefix", "You used \\openout with a prefix that doesn't belong there. I will ignore it for now.")
+  end
+  local file = scan_int()
+  scan_keyword'='
   local name = scan_filename()
   local props = {file = file, name = name}
-  if immediate == "immediate" then
+  if immediate and immediate == 8 then
     do_openout(props)
   else
     local whatsit = node.direct.new(whatsit_id, open_whatsit)
     properties[whatsit] = props
     node.direct.write(whatsit)
   end
-end, "protected")
+end, "value")
+token.luacmd("openin", function(_, prefix)
+  if prefix == "value" then return end
+  local file = scan_int()
+  scan_keyword'='
+  local name = scan_filename()
+  if ifiles[file] then
+    ifiles[file]:close()
+  end
+  local msg
+  ifiles[file] = callback_find('open_data_file', true)(name) -- raw to pick up our wrapper which handles defaults and finding the file
+end, "value")
+
 local function do_closeout(p)
   if ofiles[p.file] then
     ofiles[p.file]:close()
@@ -72,18 +113,33 @@ local function do_closeout(p)
 end
 local close_whatsit = new_whatsit('close', do_closeout)
 token.luacmd("closeout", function(_, immediate) -- \closeout
-  local file = token.scan_int()
+  if immediate == "value" then return end
+  if immediate and immediate & 0x7 ~= 0 then
+    immediate = immediate & 0x8
+    tex.error("Unexpected prefix", "You used \\closeout with a prefix that doesn't belong there. I will ignore it for now.")
+  end
+  local file = scan_int()
   local props = {file = file}
-  if immediate == "immediate" then
+  if immediate == 8 then
     do_closeout(props)
   else
     local whatsit = node.direct.new(whatsit_id, close_whatsit)
     properties[whatsit] = props
     node.direct.write(whatsit)
   end
-end, "protected")
+end, "value")
+token.luacmd("closein", function(_, prefix)
+  if prefix == "value" then return end
+  local file = scan_int()
+  if ifiles[file] then
+    ifiles[file]:close()
+    ifiles[file] = nil
+  end
+end, "value")
+
 local function do_write(p)
-  local content = token.to_string(p.data) .. '\n'
+  local data = token.serialize(p.data)
+  local content = data and data .. '\n' or '\n'
   local file = ofiles[p.file]
   if file then
     file:write(content)
@@ -93,51 +149,164 @@ local function do_write(p)
 end
 local write_whatsit = new_whatsit('write', do_write)
 token.luacmd("write", function(_, immediate) -- \write
-  local file = token.scan_int()
-  local content = token.scan_tokenlist()
+  if immediate == "value" then return end
+  if immediate and immediate & 0x7 ~= 0 then
+    immediate = immediate & 0x8
+    tex.error("Unexpected prefix", "You used \\write with a prefix that doesn't belong there. I will ignore it for now.")
+  end
+  local file = scan_int()
+  local content = scan_tokenlist()
   local props = {file = file, data = content}
-  if immediate == "immediate" then
+  if immediate == 8 then
     do_write(props)
   else
     local whatsit = node.direct.new(whatsit_id, write_whatsit)
     properties[whatsit] = props
     node.direct.write(whatsit)
   end
+end, "value")
+
+local undefined_tok = token.new(0, token.command_id'undefined_cs')
+local prefix_cmd = token.command_id'prefix'
+local function prefix_to_tokens(prefix)
+  if not prefix then return end
+  for i=2, 0, -1 do
+    if prefix & (1<<i) ~= 0 then
+      token.put_next(token.new(i, prefix_cmd))
+    end
+  end
+end
+local expand_after = token.primitive_tokens.expandafter
+local input_tok = token.primitive_tokens.input
+local endlocalcontrol = token.primitive_tokens.endlocalcontrol
+local afterassignment = token.primitive_tokens.afterassignment
+local lbrace = token.new(0, 1)
+local rbrace = token.new(0, 2)
+token.luacmd("read", function(_, prefix)
+  if immediate == "value" then return end
+  local id = scan_int()
+  if not scan_keyword'to' then
+    tex.error("Missing `to' inserted", "You should have said `\\read<number> to \\cs'.\nI'm going to look for the \\cs now.")
+  end
+  local macro = scan_csname(true)
+  local file = ifiles[id]
+  local line
+  if file then
+    line = file:reader()
+    if not line then
+      file:close()
+      ifiles[id] = nil
+    end
+  else
+    error[[FIXME: Ask the user for input]]
+  end
+  local endlocal
+  tex.runlocal(function()
+    endlocal = token.scan_next()
+    tex.sprint(endlocal)
+    tex.print(line and line ~= "" and line or " ")
+    tex.print(endlocal)
+  end)
+  local tokens = {}
+  local balance = 0
+  while true do
+    local tok = token.scan_next()
+    if tok == endlocal then break end
+    if tok.command == 1 then
+      balance = balance + 1
+    elseif tok.command == 2 then
+      balance = balance - 1
+    end
+    tokens[#tokens+1] = tok
+  end
+  if balance ~= 0 then error[[FIXME: Read additional input lines]] end
+  tex.runlocal(function()
+    tokens[#tokens+1] = rbrace
+    token.put_next(tokens)
+    token.put_next(token.primitive_tokens.def, token.create(macro), lbrace)
+    prefix_to_tokens(prefix)
+  end)
+end, "value")
+
+token.luacmd("readline", function(_, prefix)
+  if immediate == "value" then return end
+  local id = scan_int()
+  if not scan_keyword'to' then
+    tex.error("Missing `to' inserted", "You should have said `\\read<number> to \\cs'.\nI'm going to look for the \\cs now.")
+  end
+  local macro = scan_csname(true)
+  local file = ifiles[id]
+  local line
+  if file then
+    line = file:reader()
+    if not line then
+      file:close()
+      ifiles[id] = nil
+    end
+  else
+    error[[FIXME: Ask the user for input]]
+  end
+  line =  line and line:match"^(.*[^ ])[ ]*$"
+  local endlinechar = tex.endlinechar
+  if endlinechar >= 0 and endlinechar < 0x80 then
+    line = (line or '') .. string.char(endlinechar)
+  end
+  set_macro(string_catcodetable, macro, line or '', prefix)
+end, "value")
+
+local integer_code, boolean_code do
+  local value_values = token.getfunctionvalues()
+  for i=0,#value_values do
+    if value_values[i] == "integer" then
+      integer_code = i
+    elseif value_values[i] == "boolean" then
+      boolean_code = i
+    end
+  end
+end
+
+token.luacmd("ifeof", function(_)
+  local id = scan_int()
+  return boolean_code, not ifiles[id]
+end, "condition")
+
+local late_lua_whatsit = new_whatsit('late_lua', function(p, pfile, n, x, y)
+  local code = p.data
+  if not code then
+    code = token.serialize(p.token)
+  end
+  if type(code) == 'string' then
+    code = assert(load(code, nil, 't'))
+  elseif not code then
+    error[[Missing code in latelua]]
+  end
+  return pdf._latelua(pfile, x, y, code)
+end)
+token.luacmd("latelua", function() -- \latelua
+  local content = scan_tokenlist()
+  local props = {token = content}
+  local whatsit = node.direct.new(whatsit_id, late_lua_whatsit)
+  properties[whatsit] = props
+  node.direct.write(whatsit)
 end, "protected")
 
 local functions = lua.get_functions_table()
-token.luacmd("immediate", function() -- \immediate
-  local next_tok = token.scan_token()
-  if next_tok.command ~= lua_call_cmd then
-    return token.put_next(next_tok)
-  end
-  local function_id = next_tok.index
-  return functions[function_id](function_id, 'immediate')
-end, "protected")
--- functions[43] = function() -- \pdfvariable
---   local name = token.scan_string()
---   print('Missing \\pdf' .. name)
--- end
+
+require'luametalatex-meaning'
 require'luametalatex-baseregisters'
 require'luametalatex-back-pdf'
 require'luametalatex-node-luaotfload'
 
-local integer_code do
-  local value_values = token.values'value'
-  for i=0,#value_values do
-    if value_values[i] == "integer" then
-      integer_code = i
-      break
-    end
-  end
-end
 token.luacmd("Umathcodenum", function(_, scanning)
   if scanning then
-    local class, family, char = tex.getmathcodes (token.scan_int())
+    local class, family, char = tex.getmathcodes (scan_int())
     return integer_code, char | (class | family << 3) << 21
   else
-    local char = token.scan_int()
-    local mathcode = token.scan_int()
+    local char = scan_int()
+    local mathcode = scan_int()
     tex.setmathcodes(char, (mathcode >> 21) & 7, mathcode >> 24, mathcode & 0x1FFFFF)
   end
 end, "force", "global", "value")
+
+-- This is effectivly the last line before we hand over to normal TeX.
+require'luametalatex-callbacks'.__freeze = nil

luametalatex-font-enc.lua

@@ -1,3 +1,5 @@
+local readfile = require'luametalatex-readfile'
+
 local white = (lpeg.S'\0\9\10\12\13\32' + '%' * (1 - lpeg.S'\r\n')^0)^1
 local regular = 1-lpeg.S'()<>[]{}/%\0\9\10\12\13\32'
 local name = lpeg.C(regular^1)
@@ -5,8 +7,7 @@ local lname = '/' * name / 1
 local namearray = lpeg.Ct('['*white^0*lpeg.Cg(lname*white^0, 0)^-1*(lname*white^0)^0*']')
 local encfile = white^0*lname*white^0*namearray*white^0*'def'*white^0*-1
 return function(filename)
-  local file = io.open(filename)
-  local name, encoding = encfile:match(file:read'a')
-  file:close()
+  local file <close> = readfile('enc', filename)
+  local name, encoding = encfile:match(file())
   return encoding, name
 end

luametalatex-font-pk.lua

@@ -192,12 +192,9 @@ local function parse_commands(buf, off, t)
   until cmd == 245
   return off
 end
-return function(filename)
-  local f = assert(io.open(filename, 'rb'))
-  local pk = f:read'a'
-  f:close()
+return function(data)
   local res = {}
-  local off = parse_commands(pk, 1, res)
+  local off = parse_commands(data, 1, res)
   -- assert(off == #pk+1) -- TODO: Check that only fillup bytes follow
   return res
 end

luametalatex-font-resolve.lua

@@ -2,12 +2,12 @@ font.read_tfm = require'luametalatex-font-tfm'
 local read_vf = require'luametalatex-font-vf'
 font.read_vf = read_vf
 local fontmap = require'luametalatex-pdf-font-map'.fontmap
-local callback_find = callback.find
+local callbacks = require'luametalatex-callbacks'
 
 local old_font_define = font.define
 local old_addcharacters = font.addcharacters
 
-require'luametalatex-pdf-font-map'.mapfile(kpse.find_file('pdftex.map', 'map', true))
+require'luametalatex-pdf-font-map'.mapfile'pdftex.map'
 
 local all_fonts = {}
 font.fonts = all_fonts
@@ -16,9 +16,9 @@ function font.getfont(id)
 end
 
 local fontextensions = {
-  ttf = {"truetype", "truetype fonts",},
-  otf = {"opentype", "opentype fonts",},
-  pfb = {"type1", "type1 fonts",},
+  ttf = "truetype",
+  otf = "opentype",
+  pfb = "type1",
 }
 fontextensions.cff = fontextensions.otf
 local fontformats = {
@@ -60,39 +60,14 @@ function font.define(f)
     local entry = fontmap[f.name]
     if entry then
       local filename = entry[3]
-      local format = filename and filename:sub(-4, -4) == '.' and fontextensions[filename:sub(-3, -1)]
-      if format then
-        f.format = format[1]
-        f.filename = kpse.find_file(filename, format[2])
-        local encoding = entry[4]
-        if encoding then
-          f.encoding = kpse.find_file(encoding, 'enc files')
-        end
-        if entry[5] then
-          assert(special_parser:match(entry[5], 1, f))
-        end
-      else
-        local done = false
-        for _, format in ipairs(fontformats) do
-          local filename = kpse.find_file(filename, format[2])
-          if filename then
-            f.filename = filename
-            f.format = format[1]
-            local encoding = entry[4]
-            if encoding then
-              f.encoding = kpse.find_file(encoding, 'enc files')
-            end
-            if entry[5] then
-              assert(special_parser:match(entry[5], 1, f))
-            end
-            done = true
-            break
-          end
-        end
-        if not done then
-          print('!', 'type3', require'inspect'(entry))
-          f.format = "type3"
-        end
+      local format
+      if f.format == 'unknown' then
+        f.format = filename and filename:sub(-4, -4) == '.' and fontextensions[filename:sub(-3, -1)] or 'type1'
+      end
+      f.filename = filename
+      f.encoding = entry[4]
+      if entry[5] then
+        assert(special_parser:match(entry[5], 1, f))
       end
     else
       f.format = "type3"
@@ -103,7 +78,7 @@ function font.define(f)
   if f.fonts then
     for i, f in next, f.fonts do
       if not f.id then
-        f.id = assert(callback_find'define_font'(f.name, f.size or -1000))
+        f.id = assert(callbacks.define_font(f.name, f.size or -1000))
       elseif f.id == 0 then
         f.id = id
       end
@@ -121,7 +96,7 @@ function font.addcharacters(fid, newdir)
       fonts_map = {}
       for i,f in next, newdir.fonts do
         if not f.id then
-          f.id = assert(callback_find'define_font'(f.name, f.size or -1000))
+          f.id = assert(callback.define_font(f.name, f.size or -1000))
         elseif f.id == 0 then
           f.id = fid
         end

luametalatex-font-sfnt.lua

@@ -41,7 +41,7 @@ return {
   write = function(magic, tables)
     local tabdata = {}
     for t, val in next, tables do
-      tabdata[#tabdata+1] = {t, val .. string.rep("\0", (#val+3&~3)-#val), #val}
+      tabdata[#tabdata+1] = {string.format("%-4s", t), val .. string.rep("\0", (#val+3&~3)-#val), #val}
     end
     table.sort(tabdata, function(a,b)return a[1]<b[1]end)
     local logtabs = log2floor(#tabdata)

luametalatex-font-t1.lua

@@ -1,6 +1,8 @@
-local white = (lpeg.S'\0\9\10\12\13\32' + '%' * (1 - lpeg.S'\r\n')^0)^1
+local white = (lpeg.S'\0\9\10\12\13\32' + '%' * (1 - lpeg.S'\r\n')^0)^1 -- Whitespace
+
 local regular = 1-lpeg.S'()<>[]{}/%\0\9\10\12\13\32'
 local lastbase = '123456789abcdefghiklmnopqrstuvwxyz'
+
 local number = lpeg.Cmt(lpeg.R'09'^1/tonumber * '#', function(s, p, base)
                  if base < 2 then return end
                  local pattern
@@ -13,6 +15,7 @@ local number = lpeg.Cmt(lpeg.R'09'^1/tonumber * '#', function(s, p, base)
                  return p, num and tonumber(num, base)
                end)
              + (lpeg.S'+-'^-1 * ('.' * lpeg.R'09'^1 + lpeg.R'09'^1 * lpeg.P'.'^-1 * lpeg.R'09'^0) * (lpeg.S'eE' * lpeg.S'+-'^-1 * lpeg.R'09'^1)^-1)/tonumber
+
 local literalstring = lpeg.P{'(' * lpeg.Cs((
     lpeg.P'\\n'/'\n'+lpeg.P'\\r'/'\r'+lpeg.P'\\t'/'\t'+lpeg.P'\\b'/'\b'+lpeg.P'\\f'/'\f'
     +'\\'*lpeg.C(lpeg.R'07'*lpeg.R'07'^-2)/function(n)return string.char(tonumber(n, 8))end
@@ -20,16 +23,27 @@ local literalstring = lpeg.P{'(' * lpeg.Cs((
     +'\\'*lpeg.C(1)/1
     +('\n' + ('\r' * lpeg.P'\n'^-1))/'\n'
     +(1-lpeg.S'()\\')+lpeg.V(1))^0) * ')'}
+
 local hexstring = '<' * lpeg.Cs((
   lpeg.C(lpeg.R'09'+lpeg.R'af'+lpeg.R'AF')*(lpeg.C(lpeg.R'09'+lpeg.R'af'+lpeg.R'AF')+lpeg.Cc'0')/function(a,b)return string.char(tonumber(a..b, 16))end)^0) * '>'
+
 local name = lpeg.C(regular^1)
 local lname = '/' * name / 1
+
+local boolean = (lpeg.P'true' + 'false')/{["true"] = true, ["false"] = false}
+
+-- Everything above this line works pretty reliable and can be understood by reading the PostScript specs.
+
+-- This is Type1 specific. The only thing which might need adjustment is adding alternative spellings for -|, RD, |-, |, etc.
+local binary_bytes = lpeg.Cmt(number*white^-1*(lpeg.P'-| ' + 'RD '), function(s, p, l)return p+l, s:sub(p, p+l-1) end)*white^-1*(lpeg.P"|-"+"|"+"ND"+"NP")
+-- Attention: The |-, |, ND, NP already contain an implicit `def`
+
 local function decrypt(key, n, cipher)
   -- Generally you should never implement your own crypto. So we call a well known, peer reviewed,
   -- high-quality cryptographic library. --- Ha-Ha, of course we are implementing by ourselves.
   -- That might be completely unsecure, but given that the encryption keys are well known constants
   -- documented in the T1 Spec, there is no need to worry about it.
-  -- Also I do not think any cryptorgraphic library would implement this anyway, it doesn't even
+  -- Also I do not think any cryptographic library would implement this anyway, it doesn't even
   -- really deserve the term encryption.
   local decoded = {string.byte(cipher, 1,-1)}
   for i=1,#decoded do
@@ -40,66 +54,77 @@ local function decrypt(key, n, cipher)
   return string.char(table.unpack(decoded, n+1))
 end
 
--- io.stdout:write(decrypt(55665, 4, string.sub(io.stdin:read'a', 7)))
-local boolean = (lpeg.P'true' + 'false')/{["true"] = true, ["false"] = false}
-local anytype = {hexstring + literalstring + number + lname + boolean + lpeg.V(2) + name, lpeg.Ct('[' * (white^-1 * lpeg.V(1))^0 * white^-1 * ']' + '{' * (white^-1 * lpeg.V(1))^0 * white^-1 * '}' * white^-1 * lpeg.P"executeonly"^-1)}
-local dict = lpeg.Cf(lpeg.Carg(1) * lpeg.Cg(white^-1*lname*white^-1*(anytype)*white^-1*lpeg.P"readonly"^-1*white^-1*lpeg.P"noaccess"^-1*white^-1*(lpeg.P"def"+"ND"+"|-"))^0, rawset)
-local encoding = (white+anytype-("dup"*white))^0/0
+local anytype = {
+      hexstring
+    + literalstring
+    + number
+    + lname
+    + boolean
+    + lpeg.V'array'
+    + name,
+  array = lpeg.Ct( '[' * (white^-1 * lpeg.V(1))^0 * white^-1 * ']' -- Arrays have two possible syntaxes
+                 + '{' * (white^-1 * lpeg.V(1))^0 * white^-1 * '}') * (white * "executeonly")^-1
+}
+
+local function skip_until(p)
+  if type(p) == 'string' then p = p * -name end
+  return (white + anytype - p)^0/0
+end
+local skip_to_begin = skip_until'begin' * 'begin'
+
+local def_like = (lpeg.P'def' + 'ND' + '|-') * -name
+
+local encoding = '/' * lpeg.C'Encoding' * -name
+               * skip_until'dup'
                * lpeg.Cf(lpeg.Ct''
                  * lpeg.Cg("dup"*white*number*white^-1*lname*white^-1*"put"*white)^0
                  , rawset)
-               * lpeg.P"readonly"^-1*white*"def"
-local function parse_encoding(offset, str)
-  local found
-  found, offset = (encoding*lpeg.Cp()):match(str, offset)
-  return found, offset
-end
-local function parse_fontinfo(offset, str)
-  local found
-  repeat
-    found, offset = ((white+(anytype-name))^0/0*name*lpeg.Cp()):match(str, offset)
-  until found == 'begin'
-  found, offset = (dict*lpeg.Cp()):match(str, offset, {})
-  offset = (white^-1*"end"*white^-1*lpeg.P"readonly"^-1*white^-1*"def"):match(str, offset)
-  return found, offset
-end
-local binary_bytes = lpeg.Cmt(number*white^-1*(lpeg.P'-| ' + 'RD '), function(s, p, l)return p+l, s:sub(p, p+l-1) end)*white^-1*(lpeg.P"|-"+"|"+"ND"+"NP")
-local charstr = white^-1*lname*(white^-1*(anytype-lname))^0/0*white^-1
+               * ("readonly"*white)^-1 * "def"
+
+local charstr = '/' * lpeg.C'CharStrings' * -name
+            * skip_until(lname) -- sometimes we get weird stuff in between. Just make sure that we don't swallow a charname
             * lpeg.Cf(lpeg.Ct''
-              * lpeg.Cg(lname*white^-1*binary_bytes*white)^0
+              * lpeg.Cg(lname*white^-1*binary_bytes*white)^0 -- Remember: binary_bytes includes a `def`
               , rawset)
             * lpeg.P"end"*white
-local subrs = (white^-1*(anytype-("dup"*white)))^0/0*white^-1
-              * lpeg.Cf(lpeg.Ct''
-                * lpeg.Cg("dup"*white^-1*number*white^-1*binary_bytes*white)^0
-                , rawset)
-              * (lpeg.P"readonly"*white)^-1 * (lpeg.P"noaccess"*white)^-1*(lpeg.P"def"+"ND"+"|-")
+
+local subrs = '/' * lpeg.C'Subrs' * -name
+            * skip_until'dup'
+            * lpeg.Cf(lpeg.Ct''
+              * lpeg.Cg("dup"*white^-1*number*white^-1*binary_bytes*white)^0
+              , rawset)
+            * (lpeg.P"readonly"*white)^-1 * (lpeg.P"noaccess"*white)^-1*(lpeg.P"def"+"ND"+"|-")
+
+-- lpeg.V(2) == dict_entries
+local dict = skip_to_begin * lpeg.V(2) * white^-1 * 'end' * white * ('readonly' * white)^-1 * ('noaccess' * white)^-1 * def_like
+local dict_entry = encoding + subrs +
+                   '/' * lpeg.C'FontInfo' * dict +
+                   lname   -- key
+                 * white^-1
+                 * anytype -- value
+                 * ((white + anytype - (def_like + 'dict' + 'array') * -name)/0 * white^-1)^0 -- Sometimes we get Postscript code in between.
+                 * def_like
+local dict_entries = lpeg.P{
+  lpeg.Cf(lpeg.Carg(1) * lpeg.Cg(white^-1*lpeg.V(3))^0, rawset),
+  lpeg.Cf(lpeg.Ct'' * lpeg.Cg(white^-1*lpeg.V(3))^0, rawset),
+  dict_entry,
+}
 local function parse_private(offset, str)
   local mydict, found
-  repeat
-    found, offset = ((white+(anytype-name))^0/0*name*lpeg.Cp()):match(str, offset)
-  until found == 'begin'
-  mydict, offset = (dict*lpeg.Cp()):match(str, offset, {})
-  found = (white^-1*lname):match(str, offset)
-  if found == "Subrs" then
-    mydict.Subrs, offset = (subrs*lpeg.Cp()):match(str, offset)
-  end
+  offset = (skip_to_begin * lpeg.Cp()):match(str, offset)
+
+  -- Scan the dictionary
+  mydict, offset = (dict_entries*lpeg.Cp()):match(str, offset, {})
   return mydict, offset
 end
 local function continue_maintable(offset, str, mydict)
-  mydict, offset = (dict*lpeg.Cp()):match(str, offset, mydict)
+  mydict, offset = (dict_entries*lpeg.Cp()):match(str, offset, mydict)
   local found = (white^-1*lname):match(str, offset)
-  if found == "FontInfo" then
-    mydict.FontInfo, offset = parse_fontinfo(offset, str)
-    return continue_maintable(offset, str, mydict)
-  elseif found == "Encoding" then
-    mydict.Encoding, offset = parse_encoding(offset, str)
-    return continue_maintable(offset, str, mydict)
-  elseif found == "Private" then
+  if found == "Private" then -- Scanned separatly because it isn't always ended in a regular way
     mydict.Private, offset = parse_private(offset, str)
     return continue_maintable(offset, str, mydict)
-  elseif found == "CharStrings" then
-    mydict.CharStrings, offset = (charstr*lpeg.Cp()):match(str, offset)
+  elseif found == "CharStrings" then -- This could be included in normal scanning, but it is our signal to terminate
+    found, mydict.CharStrings, offset = (charstr*lpeg.Cp()):match(str, offset)
     return mydict
   else
     local newoffset = ((white+name)^1/0*lpeg.Cp()):match(str, offset)
@@ -107,24 +132,17 @@ local function continue_maintable(offset, str, mydict)
       return continue_maintable(newoffset, str, mydict)
     end
   end
-  print(str:sub(offset))
   error[[Unable to read Type 1 font]]
 end
 local function parse_maintable(offset, str)
   local found
-  repeat
-    found, offset = ((white+(anytype-name))^0/0*name*lpeg.Cp()):match(str, offset)
-  until found == 'begin'
+  offset = (skip_to_begin * lpeg.Cp()):match(str, offset)
   return continue_maintable(offset, str, {})
 end
 
-return function(filename)
-  local file = io.open(filename)
-  local _, length = string.unpack("<I2I4", file:read(6))
-  local preface = file:read(length)
-  _, length = string.unpack("<I2I4", file:read(6))
-  local private = decrypt(55665, 4, file:read(length))
-  file:close()
+return function(data)
+  local preface, private = string.unpack("<xxs4xxs4", data)
+  private = decrypt(55665, 4, private)
   local after = parse_maintable(1, preface .. private)
   local lenIV = after.Private.lenIV or 4
   local chars = after.CharStrings

luametalatex-font-t1tot2.lua

@@ -40,7 +40,7 @@ local function parse_charstring(cs, subrs, result)
         elseif cmd == 1 then -- Flex initialization
         elseif cmd == 2 then -- Flex parameter
           if result[#result-1].flex then
-            result[#result] = nil -- TODO: Warn if there were values. 
+            result[#result] = nil -- TODO: Warn if there were additional values in lastresult. 
             lastresult = result[#result] -- We keep collecting arguments
           end
           lastresult.flex = true
@@ -64,14 +64,51 @@ local function parse_charstring(cs, subrs, result)
             pending[i] = lastresult[results-numargs+i]
             lastresult[results-numargs+i] = nil
           end
-          for i = 1,#lastresult.pendingargs do
-            pending[numargs+i] = lastresult.pendingargs[i]
+          if lastresult.pendingargs then
+            for i = 1,#lastresult.pendingargs do
+              pending[numargs+i] = lastresult.pendingargs[i]
+            end
           end
           if cmd == 12 then
             lastresult.pendingargs = pending
           else
             lastresult.pendingargs = nil
-            -- TODO Translate pending to counter mask
+            local n = pending[1]
+            local i = 2
+            local groups = {}
+            for group = 1, n do
+              local current = {20}
+              local last = 0
+              while pending[i+1] > 0 do
+                last = last + pending[i]
+                current[#current+1] = {1, last, pending[i+1]}
+                last = last + pending[i+1]
+                i = i+2
+              end
+              last = last + pending[i]
+              current[#current+1] = {1, last + pending[i+1], -pending[i+1]}
+              groups[group] = current
+              i = i+2
+            end
+            n = pending[i]
+            i = i+1
+            for group = 1, n do
+              local current = groups[group] or {20}
+              local last = 0
+              while pending[i+1] > 0 do
+                last = last + pending[i]
+                current[#current+1] = {3, last, pending[i+1]}
+                last = last + pending[i+1]
+                i = i+2
+              end
+              last = last + pending[i]
+              current[#current+1] = {3, last + pending[i+1], -pending[i+1]}
+              groups[group] = current
+              i = i+2
+            end
+            assert(i == #pending+1)
+            table.move(groups, 1, #groups, #result, result) -- This overwrites lastresult
+            result[#result+1] = lastresult -- And restore lastresult
           end
         else
           error[[UNSUPPORTED Othersubr]]
@@ -132,12 +169,12 @@ local function adjust_charstring(cs) -- Here we get a not yet optimized but pars
   cs[1][1] = nil
   -- That's it for the width, now we need some hinting stuff. This would be easy, if hint replacement
   -- wouldn't require hint masks in Type2. And because we really enjoy this BS, we get counter
-  -- hinting as an additional treat... Oh, if you actually you counter hinting: Please test this
+  -- hinting as an additional treat... Oh, if you actually use counter hinting: Please test this
   -- and report back if it works, because this is pretty much untested.
-  -- TODO: Even more than that, counters are not implemented at all right now, except for [hv]stem3
   local stems = {}
-  local stem3 = {}
-  -- First iterate over the charstring, recording all hints and collecting them in stems/stem3
+  local stem3 = {20}
+  local cntrs = {}
+  -- First iterate over the charstring, recording all hints and collecting them in stems/stem3/cntrs
   for i, cmd in ipairs(cs) do
     if cmd[1] == 1 or cmd[1] == 3 then
       stems[#stems + 1] = cmd
@@ -148,6 +185,9 @@ local function adjust_charstring(cs) -- Here we get a not yet optimized but pars
       stems[#stems + 1] = {c, cmd[6], cmd[7]}
       table.move(stems, #stems-2, #stems, #stem3+1, stem3)
       cs[i] = false
+    elseif cmd[1] == 20 then
+      cntrs[#cntrs+1] = cmd
+      table.move(cmd, 2, #cmd, #stems+1, stems)
     end
   end
   table.sort(stems, function(first, second)
@@ -168,18 +208,25 @@ local function adjust_charstring(cs) -- Here we get a not yet optimized but pars
       stems[i].idx = j
     end
   end
-  -- Now the indices are known, so the cntrmask can be written, if stem3 occured.
+  -- Now the indices are known, so the cntrmask can be written, if counters or stem3 occured.
   -- This is done before writing the stem list to make the thable.insert parameters easier.
+  -- First translate stem3 into a counter group
+  if stem3[2] then
+    cntrs[#cntrs+1] = stem3
+    table.insert(cs, 2, stem3)
+  end
   local bytes = {}
-  if stem3[1] then
-      for l = 1, math.floor((j + 7)/8) do
-        bytes[l] = 0
-      end
-    for l = 1, #stem3 do
-      local idx = stem3[l].idx-1
-      bytes[math.floor(idx/8) + 1] = bytes[math.floor(idx/8) + 1] | (1<<(7-idx%8))
+  for i=1, #cntrs do
+    local cntr = cntrs[i]
+    for l = 1, math.floor((j + 7)/8) do
+      bytes[l] = 0
     end
-    table.insert(cs, 2, {20, string.char(table.unpack(bytes))})
+    for l = 2, #cntr do
+      local idx = cntr[l].idx-1
+      bytes[math.floor(idx/8) + 1] = bytes[math.floor(idx/8) + 1] | (1<<(7-idx%8))
+      cntr[l] = nil
+    end
+    cntr[2] = string.char(table.unpack(bytes))
   end
   local current = 1
   -- Then list the collected stems at the beginning of the charstring
@@ -220,7 +267,7 @@ local function adjust_charstring(cs) -- Here we get a not yet optimized but pars
         cs[i] = false
         i = i+1
       end
-      for l = 1, #stem3 do
+      for l = 2, #stem3 do
         local idx = stem3[l].idx-1
         bytes[math.floor(idx/8) + 1] = bytes[math.floor(idx/8) + 1] | (1<<(7-idx%8))
       end

luametalatex-font-tfm.lua

@@ -1,3 +1,5 @@
+local readfile = require'luametalatex-readfile'
+
 local upper_mask = (1<<20)-1<<44
 local shifted_sign = 1<<43
 local function scale(factor1, factor2)
@@ -141,13 +143,9 @@ local function parse_tfm(buf, i, size)
 end
 local basename = ((1-lpeg.S'\\/')^0*lpeg.S'\\/')^0*lpeg.C((1-lpeg.P'.tfm'*-1)^0)
 return function(name, size)
-  local filename = kpse.find_file(name, 'tfm', true)
-  if not filename then return end
-  local f = io.open(filename)
-  if not f then return end
-  local buf = f:read'*a'
-  f:close()
-  local result = parse_tfm(buf, 1, size)
+  local file <close> = readfile('tfm', name)
+  if not file then return end
+  local result = parse_tfm(file(), 1, size)
   result.name = basename:match(name)
   return result
 end

luametalatex-font-ttf-data.lua

@@ -0,0 +1,2 @@
+return {[0] =
+".notdef", ".null", "nonmarkingreturn", "space", "exclam", "quotedbl", "numbersign", "dollar", "percent", "ampersand", "quotesingle", "parenleft", "parenright", "asterisk", "plus", "comma", "hyphen", "period", "slash", "zero", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine", "colon", "semicolon", "less", "equal", "greater", "question", "at", "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z", "bracketleft", "backslash", "bracketright", "asciicircum", "underscore", "grave", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z", "braceleft", "bar", "braceright", "asciitilde", "Adieresis", "Aring", "Ccedilla", "Eacute", "Ntilde", "Odieresis", "Udieresis", "aacute", "agrave", "acircumflex", "adieresis", "atilde", "aring", "ccedilla", "eacute", "egrave", "ecircumflex", "edieresis", "iacute", "igrave", "icircumflex", "idieresis", "ntilde", "oacute", "ograve", "ocircumflex", "odieresis", "otilde", "uacute", "ugrave", "ucircumflex", "udieresis", "dagger", "degree", "cent", "sterling", "section", "bullet", "paragraph", "germandbls", "registered", "copyright", "trademark", "acute", "dieresis", "notequal", "AE", "Oslash", "infinity", "plusminus", "lessequal", "greaterequal", "yen", "mu", "partialdiff", "summation", "product", "pi", "integral", "ordfeminine", "ordmasculine", "Omega", "ae", "oslash", "questiondown", "exclamdown", "logicalnot", "radical", "florin", "approxequal", "Delta", "guillemotleft", "guillemotright", "ellipsis", "nonbreakingspace", "Agrave", "Atilde", "Otilde", "OE", "oe", "endash", "emdash", "quotedblleft", "quotedblright", "quoteleft", "quoteright", "divide", "lozenge", "ydieresis", "Ydieresis", "fraction", "currency", "guilsinglleft", "guilsinglright", "fi", "fl", "daggerdbl", "periodcentered", "quotesinglbase", "quotedblbase", "perthousand", "Acircumflex", "Ecircumflex", "Aacute", "Edieresis", "Egrave", "Iacute", "Icircumflex", "Idieresis", "Igrave", "Oacute", "Ocircumflex", "apple", "Ograve", "Uacute", "Ucircumflex", "Ugrave", "dotlessi", "circumflex", "tilde", "macron", "breve", "dotaccent", "ring", "cedilla", "hungarumlaut", "ogonek", "caron", "Lslash", "lslash", "Scaron", "scaron", "Zcaron", "zcaron", "brokenbar", "Eth", "eth", "Yacute", "yacute", "Thorn", "thorn", "minus", "multiply", "onesuperior", "twosuperior", "threesuperior", "onehalf", "onequarter", "threequarters", "franc", "Gbreve", "gbreve", "Idotaccent", "Scedilla", "scedilla", "Cacute", "cacute", "Ccaron", "ccaron", "dcroat"}

luametalatex-font-vf.lua

@@ -1,3 +1,5 @@
+local readfile = require'luametalatex-readfile'
+
 local fontcmds = {
   [243] = ">I1I4I4I4BB",
   [244] = ">I2I4I4I4BB",
@@ -166,13 +168,9 @@ local function parse_vf(buf, i, size)
 end
 local basename = ((1-lpeg.S'\\/')^0*lpeg.S'\\/')^0*lpeg.C((1-lpeg.P'.tfm'*-1)^0)
 return function(name, size, must_exist)
-  local filename = kpse.find_file(name, 'vf', must_exist)
-  if not filename then return end
-  local f = io.open(filename)
-  if not f then return end
-  local buf = f:read'*a'
-  f:close()
-  local result = parse_vf(buf, 1, size)
+  local file <close> = readfile('vf', name)
+  if not file then return end
+  local result = parse_vf(file(), 1, size)
   result.name = basename:match(name)
   return result
 end

luametalatex-init-config.lua

@@ -32,3 +32,4 @@ for _, name in ipairs{
   local value = var_value(name)
   texconfig[name] = tonumber(value) or value or nil
 end
+texconfig.token_size = 10000000

luametalatex-init.lua

@@ -1,30 +1,41 @@
 do
-  local ourpath = lua.startupfile:match('(.*[/\\])[^/\\]*%.lua$')
-  kpse = assert(package.loadlib(ourpath .. 'kpse.so', 'luaopen_kpse'))()
+  local ourpath
+  ourpath, texconfig.formatname = lua.startupfile:match('(.*[/\\])([^/\\]*)%-init%.lua$')
+  local function try_lib(name)
+    local path = string.format('%s%s.%s', ourpath, name,
+      os.type == 'windows' and 'dll' or 'so')
+    return package.loadlib(path, '*') and path
+  end
+  local library  = try_lib'luametalatex' or try_lib'kpse'
+  if not library then
+    error[[C support library not found. Please fix your installation]]
+  end
+  kpse = assert(package.loadlib(library, 'luaopen_luametalatex_kpse') or package.loadlib(library, 'luaopen_kpse'))()
+  package.loaded.kpse = kpse
+  package.preload.luaharfbuzz = package.loadlib(library, 'luaopen_luametalatex_harfbuzz') or package.loadlib(library, 'luaopen_luametalatex_harfbuzz') or nil
 end
-local interaction
 do
-  local arg0, progname
+  local arg_pattern = '-' * lpeg.P'-'^-1 * lpeg.C((1-lpeg.P'=')^1) * ('=' * lpeg.C(lpeg.P(1)^0) + lpeg.Cc(true))
   for _, a in ipairs(arg) do
-    if a:sub(1,11) == "--progname=" then
-      progname = a:sub(12)
-    elseif a:sub(1,7) == "--arg0=" then
-      arg0 = a:sub(8)
-    elseif a:match'^%-%-?interaction=' then
-      local interaction_name = a:sub(a:find'='+1)
-      interaction = ({
-        batchmode=0,
-        nonstopmode=1,
-        scrollmode=2,
-        errorstopmode=3,
-      })[interaction_name]
-      if not interaction then
-        texio.write('term', string.format('Unknown interaction mode %q ignored.\n', interaction_name))
-      end
+    local name, value = arg_pattern:match(a)
+    if name then
+      arg[name] = math.tointeger(value) or value
     end
   end
-  os.arg0 = arg0
-  kpse.set_program_name(arg0, progname)
+end
+kpse.set_program_name(arg.arg0 or arg[arg[0]], arg.progname)
+do
+  local interaction = ({ [true] = 3, [false] = false,
+    batchmode=0,
+    nonstopmode=1,
+    scrollmode=2,
+    errorstopmode=3,
+  })[arg.interaction or false]
+  if interaction then
+    tex.setinteraction(interaction)
+  elseif interaction == nil then
+    texio.write('term', string.format('Unknown interaction mode %q ignored.\n', arg.interaction))
+  end
 end
 package.searchers[2] = function(modname)
   local filename = kpse.find_file(modname, "lua", true)
@@ -42,6 +53,7 @@ kpse.set_maketex("pk", true, "compile")
 require'luametalatex-init-config'
 local callback_register = callback.register
 local build_bytecode
+status.ini_version = status.run_state == 0
 if status.ini_version then
   local build_bytecode_mod = require'luametalatex-build-bytecode'
   local preloaded_modules = {}
@@ -57,11 +69,9 @@ if status.ini_version then
   end
 end
 
-callback_register('find_format_file', function(name) return kpse.find_file(name, 'fmt', true) end)
+callback_register('find_format_file', function(name) texconfig.formatname = kpse.find_file(name, 'fmt', true) return texconfig.formatname end)
+-- texconfig.firstline = [[\show ]]
 function texconfig.init()
-  if interaction then
-    tex.setinteraction(interaction)
-  end
   if build_bytecode then -- Effectivly if status.ini_version
     require'luametalatex-lateinit'(build_bytecode)
   else

luametalatex-lateinit.lua

@@ -13,7 +13,33 @@ pdf = {
   variable = {},
 }
 require'luametalatex-font-resolve' -- Replace font.define. Must be loaded before callbacks
-require'luametalatex-callbacks'
+require'luametalatex-basecallbacks'
+local callbacks = require'luametalatex-callbacks'
+
+local function swap_table(t)
+  local s = {}
+  for k, v in next, t do
+    s[v] = k
+  end
+  return s
+end
+
+local primitives = {}
+do
+  local token_primitives = token.getprimitives()
+  local token_new = token.new
+  for i=1,#token_primitives do
+    local prim = token_primitives[i]
+    primitives[prim[3]] = token_new(prim[2], prim[1])
+  end
+end
+token.primitive_tokens = primitives
+
+do
+  local command_id = swap_table(token.getcommandvalues())
+  function token.command_id(name) return command_id[name] end
+end
+token.value = swap_table(token.getfunctionvalues())
 
 local functions = lua.getfunctionstable()
 -- I am not sure why this is necessary, but otherwise LuaMetaTeX resets
@@ -43,6 +69,7 @@ local undefined_cmd = token.command_id'undefined_cs'
 local lua_call_cmd = token.command_id'lua_call'
 local lua_value_cmd = token.command_id'lua_value'
 local lua_expandable_call_cmd = token.command_id'lua_expandable_call'
+local if_test_cmd = token.command_id'if_test'
 function token.luacmd(name, func, ...)
   local idx
   local tok = token.create(name)
@@ -53,6 +80,8 @@ function token.luacmd(name, func, ...)
     idx = tok.index
   elseif cmd == lua_expandable_call_cmd then
     idx = tok.index
+  elseif cmd == if_test_cmd and tok.index > 48 then
+    idx = tok.index - 48
   elseif ... == 'force' then
     idx = new_luafunction(name)
     set_lua(name, idx, select(2, ...))
@@ -71,15 +100,18 @@ end
 
 if initex then
   local build_bytecode = nil -- To be filled
-  callback.register('pre_dump', function()
+  function callbacks.pre_dump()
+    local user_callback = callbacks.pre_dump
+    if user_callback then user_callback() end
+
     local prepared = lua.prepared_code
     prepared[1] = string.format("fixupluafunctions(%i)", predefined_luafunctions)
     for i=0,0 do -- maybeFIXME: In practise only one language is preloaded in LuaTeX anyway
     -- for i=0,tex.count[19] do -- Sometimes catches reserved language ids which are not used yet
     -- for i=0,lang.new():id()-1 do -- lang.new():id() is always 0 in luametatex?!?
-      local l = lang.new(i)
+      local l = language.new(i)
       local str = string.format("do \z
-        local l = lang.new(%i)\z
+        local l = language.new(%i)\z
         l:hyphenationmin(%i)\z
         l:prehyphenchar(%i)\z
         l:posthyphenchar(%i)\z
@@ -110,7 +142,8 @@ if initex then
       end
     end
     lua.bytecode[tex.count[262]+1] = build_bytecode(table.concat(prepared, '\n'))
-  end)
+  end
+  callbacks.__freeze('pre_dump', true)
   return function(f)
     build_bytecode = f
     return require'luametalatex-firstcode'

luametalatex-local.lua

@@ -12,9 +12,9 @@ lua.get_functions_table()[restore_func] = function()
     end
   end
 end
-local restore_toks = {token.new(2, token.command_id'after_something') , token.new(restore_func, token.command_id'lua_call')} -- \atendofgroup FIXME: Detect mode automatically once token.primitive is fixed
+local restore_toks = {token.primitive_tokens.atendofgroup , token.new(restore_func, token.command_id'lua_call')}
 local put_next = token.put_next
-local runtoks = tex.runtoks
+local runlocal = tex.runlocal
 local function put_restore_toks()
   put_next(restore_toks)
 end
@@ -35,7 +35,7 @@ return function(t, k, v, global)
     local level = stack[l]
     if not level then
       level = {}
-      runtoks(put_restore_toks)
+      runlocal(put_restore_toks)
       stack[l] = level
     end
 

luametalatex-ltexpl-hook.tex

@@ -1,6 +1,10 @@
 \directlua{unhook_expl()}
 % See baseregisters for list of toks pdfvariables
 \directlua{initialize_pdf_toks()}
+% We hardcode the ids of two catcodetables. If they ever change in the format, abort.
+\if 0\ifnum1=\catcodetable@initex\else\expandafter1\fi\ifnum2=\catcodetable@string\else\expandafter1\fi 0\else
+  \errmessage{Inconsistent catcodetable identifiers}
+\fi
 \ifx\@tfor\undefined
   \def\@tfor#1\do#2{}
 \fi

luametalatex-microtype.sty

@@ -24,10 +24,10 @@
   \noexpand\luametalatex@@everyjobandnow{\noexpand\directlua{!
     lua.get_functions_table()[\the\luametalatex@@expandglyphsinfont] = function()
       token.put_next(token.create'fontid')
-      local font = token.scan_int()
-      local stretch = token.scan_int()
-      local shrink = token.scan_int()
-      local step = token.scan_int()
+      local font = token.scan_integer()
+      local stretch = token.scan_integer()
+      local shrink = token.scan_integer()
+      local step = token.scan_integer()
       token.set_macro('pickup@font@@hook@luametalatex@microtype@' .. font, string.format("{}{%i}{%i}{%i}", stretch, shrink, step), "global")
     end
   }}

luametalatex-nodewriter.lua

@@ -30,6 +30,7 @@ local getchar = direct.getchar
 local rangedimensions = direct.rangedimensions
 local traverse_id = direct.traverse_id
 local getdata = direct.getdata
+local getorientation = direct.getorientation
 
 local utils = require'luametalatex-pdf-utils'
 local strip_floats = utils.strip_floats
@@ -39,6 +40,8 @@ local make_resources = require'luametalatex-pdf-resources'
 local pdf_font_map = require'luametalatex-pdf-font-deduplicate'
 local get_whatsit_handler = require'luametalatex-whatsits'.handler
 
+local write_matrix -- Defined later
+
 local dir_id = node.id'dir'
 
 local function doublekeyed(t, id2name, name2id, index)
@@ -162,6 +165,107 @@ local function toglyph(p, fid, x, y, exfactor)
   p.mode = glyph
   p.pending[1] = "[("
 end
+
+local function boxrotation(p, list, x, y)
+  local orientation, xoff, yoff, woff, hoff, doff = getorientation(list)
+  if not orientation then return x, y, direct.getwhd(list) end
+  x, y = x + xoff, y + yoff
+  local baseorientation = orientation & 0xF
+  local v_anchor = (orientation & 0xF0) >> 4
+  local h_anchor = (orientation & 0xF00) >> 8
+  -- assert(baseorientation & 8 == 0)
+  if baseorientation & 4 == 4 then
+    -- assert(baseorientation < 6)
+    baseorientation, v_anchor = 0, baseorientation - 3
+  end
+  if baseorientation & 1 == 0 then -- horizontal
+    if h_anchor == 0 then
+    elseif h_anchor == 1 then
+      x = x - woff
+    elseif h_anchor == 2 then
+      x = x + woff
+    elseif h_anchor == 3 then
+      x = x - woff//2
+    elseif h_anchor == 4 then
+      x = x + woff//2
+    -- else assert(false)
+    end
+    local flipped = baseorientation ~= 0
+    if v_anchor ~= 0 then
+      local h, d = hoff, doff
+      if flipped then h, d = d, h end
+      if v_anchor == 1 then
+        y = y + d
+      elseif v_anchor == 2 then
+        y = y - h
+      elseif v_anchor == 3 then
+        y = y + (d - h)//2
+    -- else assert(false)
+      end
+    end
+    if flipped then
+      write_matrix(-1, 0, 0, -1, 2*x, 2*y, p)
+      x, y = x - woff, y
+    end
+  else -- vertical
+    if v_anchor == 0 then
+    elseif v_anchor == 1 then
+      y = y + woff
+    elseif v_anchor == 2 then
+      y = y - woff
+    elseif v_anchor == 3 then
+      y = y - woff//2
+    -- else assert(false)
+    end
+    local flipped = baseorientation ~= 1
+    if h_anchor ~= 0 then
+      local h, d = hoff, doff
+      if flipped then h, d = d, h end
+      if h_anchor == 1 then
+        x = x - h - d
+      elseif h_anchor == 2 then
+        x = x + h + d
+      elseif h_anchor == 3 then
+        x = x - (d + h)//2
+      elseif h_anchor == 4 then
+        x = x + (d + h)//2
+      elseif h_anchor == 5 then
+        x = x - d
+      elseif h_anchor == 6 then
+        x = x + h
+    -- else assert(false)
+      end
+    end
+    if flipped then
+      write_matrix(0, 1, -1, 0, x+y, y-x, p)
+      x, y = x, y - hoff
+    else
+      write_matrix(0, -1, 1, 0, x-y, x+y, p)
+      x, y = x - woff, y + doff
+    end
+  end
+  return x, y, woff, hoff, doff
+end
+
+local function endboxrotation(p, list, x, y)
+  local orientation, xoff, yoff, woff, hoff, doff = getorientation(list)
+  if not orientation then return end
+  local orientation = orientation & 0xF
+  -- assert(orientation & 8 == 0)
+  if orientation & 4 == 4 or orientation == 0 then
+    -- write_matrix(1, 0, 0, 1, 0, 0, p)
+  elseif orientation == 1 then
+    x, y = x + woff, y - doff
+    write_matrix(0, 1, -1, 0, x+y, y-x, p)
+  elseif orientation == 2 then
+    x = x + woff
+    write_matrix(-1, 0, 0, -1, 2*x, 2*y, p)
+  elseif orientation == 3 then
+    y = y + hoff
+    write_matrix(0, -1, 1, 0, x-y, x+y, p)
+  end
+end
+
 -- Let's start with "handlers" for nodes which do not need any special handling:
 local function ignore_node() end
 -- The following are already handled by the list handler because they only correspond to blank space:
@@ -182,9 +286,12 @@ function nodehandler.hlist(p, list, x0, y, outerlist, origin, level)
       x0 = x0 + getshift(list)
     end
   end
+  local width
+  x0, y, width = boxrotation(p, list, x0, y)
+  local x = x0
   local direction = getdirection(list)
   if direction == 1 then
-    x0 = x0 + getwidth(list)
+    x = x + width
   end
   local dirstack = {}
   local dirnodes = {}
@@ -200,7 +307,6 @@ function nodehandler.hlist(p, list, x0, y, outerlist, origin, level)
   for i=1,#dirstack do
     dirnodes[dirstack[i]] = rangedimensions(list, dirstack[i])
   end
-  local x = x0
   local linkcontext = p.linkcontext
   if linkcontext then
     linkcontext:set(p, x, y, list, level+1, 'start')
@@ -227,7 +333,7 @@ function nodehandler.hlist(p, list, x0, y, outerlist, origin, level)
       local next = getnext(n)
       local w = next and rangedimensions(list, n, next) or rangedimensions(list, n)
       if direction == 1 then x = x - w end
-      nodehandler[id](p, n, x, y, list, x0, level+1)
+      nodehandler[id](p, n, x, y, list, x0, level+1, direction)
       if direction == 0 then x = w + x end
     end
   end
@@ -235,6 +341,7 @@ function nodehandler.hlist(p, list, x0, y, outerlist, origin, level)
   if linkcontext then
     linkcontext:set(p, x, y, list, level+1, 'end')
   end
+  endboxrotation(p, list, x0, y)
 end
 function nodehandler.vlist(p, list, x, y0, outerlist, origin, level)
   if outerlist then
@@ -244,17 +351,20 @@ function nodehandler.vlist(p, list, x, y0, outerlist, origin, level)
       x = x + getshift(list)
     end
   end
-  y0 = y0 + getheight(list)
+  local width, height
+  x, y0, width, height = boxrotation(p, list, x, y0)
   local y = y0
+  y = y + height
   for n in traverse(getlist(list)) do
     local d, h, _ = 0, direct.effective_glue(n, list) or math.tointeger(getkern(n))
     if not h then
       _, h, d = direct.getwhd(n)
     end
     y = y - (h or 0)
-    nodehandler[getid(n)](p, n, x, y, list, y0, level+1)
+    nodehandler[getid(n)](p, n, x, (y+.5)//1, list, y0, level+1)
     y = y - (d or 0)
   end
+  endboxrotation(p, list, x, y0)
 end
 do
 local rulesubtypes = {}
@@ -293,7 +403,10 @@ function nodehandler.rule(p, n, x, y, outer)
   end
 end
 end
-function nodehandler.disc(p, n, x, y, list, ...) -- FIXME: I am not sure why this can happen, let's assume we can use .replace
+-- If we encounter disc here, we can just use .replace. We could make TeX drop these using node.flatten_discretionaries,
+-- but for now we just accept them. This approach might be a bit faster, but it leads to a few issue due to directions etc.
+-- so it might change soon(ish) TODO: Review
+function nodehandler.disc(p, n, x, y, list, ...)
   for n in traverse(getreplace(n)) do
     local next = getnext(n)
     local w = next and rangedimensions(list, n, next) or rangedimensions(list, n)
@@ -301,9 +414,17 @@ function nodehandler.disc(p, n, x, y, list, ...) -- FIXME: I am not sure why thi
     x = w + x
   end
 end
+local gluesubtypes = {}
+for i, n in next, node.subtypes'glue' do
+  gluesubtypes[n] = i
+end
+local  leaders_glue = gluesubtypes.leaders
+local cleaders_glue = gluesubtypes.cleaders
+local xleaders_glue = gluesubtypes.xleaders
+local gleaders_glue = gluesubtypes.gleaders
 function nodehandler.glue(p, n, x, y, outer, origin, level) -- Naturally this is an interesting one.
   local subtype = getsubtype(n)
-  if subtype < 100 then return end -- We only really care about leaders
+  if subtype < leaders_glue then return end -- We only really care about leaders
   local leader = getleader(n)
   local w = direct.effective_glue(n, outer)
   if getid(leader) == 2 then -- We got a rule, this should be easy
@@ -319,7 +440,7 @@ function nodehandler.glue(p, n, x, y, outer, origin, level) -- Naturally this is
   if getid(outer) ~= 0 then
     y = y + w
   end
-  if subtype == 100 then
+  if subtype == leaders_glue then
     if getid(outer) == 0 then
       local newx = ((x-origin - 1)//lwidth + 1) * lwidth + origin
       -- local newx = -(origin-x)//lwidth * lwidth + origin
@@ -331,14 +452,14 @@ function nodehandler.glue(p, n, x, y, outer, origin, level) -- Naturally this is
       w = w + newy - y
       y = newy
     end
-  elseif subtype == 101 then
+  elseif subtype == cleaders_glue then
     local inner = w - (w // lwidth) * lwidth
     if getid(outer) == 0 then
       x = x + inner/2
     else
       y = y - inner/2
     end
-  elseif subtype == 102 then
+  elseif subtype == xleaders_glue then
     local count = w // lwidth
     local skip = (w - count * lwidth) / (count + 1)
     if getid(outer) == 0 then
@@ -347,7 +468,7 @@ function nodehandler.glue(p, n, x, y, outer, origin, level) -- Naturally this is
       y = y - skip
     end
     lwidth = lwidth + skip
-  elseif subtype == 103 then
+  elseif subtype == gleaders_glue then
     if getid(outer) == 0 then
       local newx = ((x - 1)//lwidth + 1) * lwidth
       w = w + x - newx
@@ -375,46 +496,48 @@ function nodehandler.glue(p, n, x, y, outer, origin, level) -- Naturally this is
   end
 end
 
+local vf_state
+
 local pdf_escape = require'luametalatex-pdf-escape'.escape_raw
 local match = lpeg.match
-local function do_commands(p, c, f, fid, x, y, outer, ...)
+local function do_commands(p, c, f, cid, fid, x, y, outer, x0, level, direction)
   local fonts = f.fonts
-  local stack, current_font = {}, fonts[1]
+  local stack, current_font = {}, fonts and fonts[1] and fonts[1].id or fid
   for _, cmd in ipairs(c.commands) do
     if cmd[1] == "node" then
       local cmd = cmd[2]
       assert(node.type(cmd))
       cmd = todirect(cmd)
-      nodehandler[getid(cmd)](p, cmd, x, y, nil, ...)
+      nodehandler[getid(cmd)](p, cmd, x, y, outer, x0, level, direction)
       x = x + getwidth(cmd)
     elseif cmd[1] == "font" then
-      current_font = assert(fonts[cmd[2]], "invalid font requested")
+      current_font = assert(fonts[cmd[2]], "invalid font requested").id
     elseif cmd[1] == "char" then
       local n = direct.new'glyph'
       setsubtype(n, 256)
-      setfont(n, current_font.id, cmd[2])
-      nodehandler.glyph(p, n, x, y, outer, ...)
+      setfont(n, current_font, cmd[2])
+      nodehandler.glyph(p, n, x, y, outer, x0, level, direction)
       x = x + getwidth(n)
       direct.free(n)
     elseif cmd[1] == "slot" then
-      current_font = assert(fonts[cmd[2]], "invalid font requested")
+      current_font = assert(fonts[cmd[2]], "invalid font requested").id
       local n = direct.new'glyph'
       setsubtype(n, 256)
-      setfont(n, current_font.id, cmd[3])
-      nodehandler.glyph(p, n, x, y, outer, ...)
+      setfont(n, current_font, cmd[3])
+      nodehandler.glyph(p, n, x, y, outer, x0, level, direction)
       x = x + getwidth(n)
       direct.free(n)
     elseif cmd[1] == "rule" then
       local n = direct.new'rule'
-      setheight(n, cmd[2])
-      setwidth(n, cmd[3])
-      nodehandler.rule(p, n, x, y, outer, ...)
+      setheight(n, cmd[3])
+      setwidth(n, cmd[2])
+      nodehandler.rule(p, n, x, y, outer, x0, level, direction)
       x = x + getwidth(n)
       direct.free(n)
-    elseif cmd[1] == "left" then
+    elseif cmd[1] == "right" then
       x = x + cmd[2]
     elseif cmd[1] == "down" then
-      y = y + cmd[2]
+      y = y - cmd[2]
     elseif cmd[1] == "push" then
       stack[#stack + 1] = {x, y}
     elseif cmd[1] == "pop" then
@@ -424,12 +547,18 @@ local function do_commands(p, c, f, fid, x, y, outer, ...)
     elseif cmd[1] == "special" then
       error[[specials aren't supported yet]] -- TODO
     elseif cmd[1] == "pdf" then
-      pdf.write(cmd[3] and cmd[2] or "origin", cmd[3], x, y, p)
+      local mode, literal = cmd[2], cmd[3]
+      if not literal then mode, literal = "origin", mode end
+      pdf.write(mode, literal, x, y, p)
     elseif cmd[1] == "lua" then
       cmd = cmd[2]
       if type(cmd) == "string" then cmd = load(cmd) end
       assert(type(cmd) == "function")
-      pdf._latelua(p, x, y, cmd, fid, c)
+      local old_vf_state = vf_state -- This can be triggered recursivly in odd cases
+      vf_state = {p, stack, current_font, x, y, outer, x0, level, direction}
+      pdf._latelua(p, x, y, cmd, fid, cid)
+      current_font, x, y = vf_state[3], vf_state[4], vf_state[5]
+      vf_state = old_vf_state
     elseif cmd[1] == "image" then
       error[[images aren't supported yet]] -- TODO
       -- ???
@@ -438,20 +567,77 @@ local function do_commands(p, c, f, fid, x, y, outer, ...)
     end
   end
 end
-function nodehandler.glyph(p, n, x, y, ...)
+-- Now we basically duplicate all of that for the `vf` library...
+vf = {
+  char = function(cid)
+    local n = direct.new'glyph'
+    setsubtype(n, 256)
+    setfont(n, vf_state[3], cid)
+    local x = vf_state[4]
+    nodehandler.glyph(vf_state[1], n, x, vf_state[5], vf_state[6], vf_state[7], vf_state[8], vf_state[9])
+    vf_state[4] = x + getwidth(n)
+    direct.free(n)
+  end,
+  down = function(dy)
+    vf_state[5] = vf_state[5] - dy
+  end,
+  fontid = function(fid)
+    vf_state[3] = fid
+  end,
+  -- image = function(img) -- TODO
+  node = function(n)
+    assert(node.type(n))
+    cmd = todirect(n)
+    local x = vf_state[4]
+    nodehandler[getid(n)](vf_state[1], n, x, vf_state[5], vf_state[6], vf_state[7], vf_state[8], vf_state[9])
+    vf_state[4] = x + getwidth(n)
+  end,
+  nop = function() end,
+  pop = function()
+    local stack = vf_state[2]
+    local top = stack[#stack]
+    stack[#stack] = nil
+    vf_state[4], vf_state[5] = top[1], top[2]
+  end,
+  push = function()
+    local stack = vf_state[2]
+    stack[#stack + 1] = vf_state[4], vf_state[5]
+  end,
+  right = function(dx)
+    vf_state[4] = vf_state[4] + dx
+  end,
+  rule = function(width, height)
+    local n = direct.new'rule'
+    setheight(n, height)
+    setwidth(n, width)
+    local x = vf_state[4]
+    nodehandler.rule(vf_state[1], n, x, vf_state[5], vf_state[6], vf_state[7], vf_state[8], vf_state[9])
+    vf_state[4] = x + getwidth(n)
+    direct.free(n)
+  end,
+  special = function()
+    error[[specials aren't supported yet]] -- TODO
+  end,
+  pdf = function(mode, literal)
+    if not literal then mode, literal = "origin", mode end
+    pdf.write(mode, literal, vf_state[4], vf_state[5], vf_state[1])
+  end,
+}
+function nodehandler.glyph(p, n, x, y, outer, x0, level, direction)
   if getfont(n) ~= p.vfont.fid then
     p.vfont.fid = getfont(n)
     p.vfont.font = font.getfont(getfont(n)) or font.fonts[getfont(n)]
   end
   local f, fid = p.vfont.font, p.vfont.fid
-  local c = f.characters[getchar(n)]
+  local cid = getchar(n)
+  local c = f.characters[cid]
   if not c then
     texio.write_nl("Missing character")
     return
   end
-  if c.commands then return do_commands(p, c, f, fid, x, y, ...) end
+  if c.commands then return do_commands(p, c, f, cid, fid, x, y, outer, x0, level, direction) end
   local xoffset, yoffset = getoffsets(n)
-  toglyph(p, getfont(n), x + xoffset, y + yoffset, getexpansion(n))
+  toglyph(p, getfont(n), x + (direction == 1 and -xoffset or xoffset), y + yoffset, getexpansion(n))
   local index = c.index
   if index then
     -- if f.encodingbytes == -3 then
@@ -490,7 +676,7 @@ function pdf._latelua(p, x, y, func, ...)
   global_p, global_x, global_y = p, x, y
   return func(...)
 end
-function pdf.write_matrix(a, b, c, d, e, f, p)
+function write_matrix(a, b, c, d, e, f, p)
   e, f, p = e or 0, f or 0, p or global_p
   local pending = p.pending_matrix
   if p.mode ~= cm_pending then
@@ -503,7 +689,7 @@ function pdf.write_matrix(a, b, c, d, e, f, p)
   end
   pending[1], pending[2], pending[3], pending[4], pending[5], pending[6] = a, b, c, d, e, f
 end
-local write_matrix = pdf.write_matrix
+pdf.write_matrix = write_matrix
 local literal_type_names = { [0] =
   'origin', 'page', 'direct', 'raw', 'text'
 }

luametalatex-pdf-font-cff.lua

@@ -1,3 +1,5 @@
+local readfile = require'luametalatex-readfile'
+
 local sfnt = require'luametalatex-font-sfnt'
 local stdStrings = require'luametalatex-font-cff-data'
 local offsetfmt = ">I%i"
@@ -449,11 +451,6 @@ function myfunc(buf, i0, fontid, usedcids, encoding, trust_widths)
   end
   -- top.CharStrings = named_charstrings
   if not top.ROS then
-    -- if encoding == true and top.Encoding < 3 then
-      -- if not reencode and parsed_t1.Encoding == "StandardEncoding" then
-      --   reencode = kpse.find_file("8a.enc", "enc files")
-      -- end
-    -- end
     if encoding == true then -- Use the built-in encoding
       CharStrings = parse_encoding(buf, i0, top.Encoding, CharStrings)
     elseif encoding then
@@ -504,8 +501,7 @@ function myfunc(buf, i0, fontid, usedcids, encoding, trust_widths)
     parse_fdselect(buf, i0+top.FDSelect, CharStrings)
   end
   local glyphs = {}
-  -- if false and usedcids then -- Subsetting FIXME: Disabled, because other tables have to be fixed up first
-  if usedcids then -- Subsetting FIXME: Should be Disabled, because other tables have to be fixed up first -- Actually seems to work now, let's test it a bit more
+  if usedcids then -- Subsetting maybeFIXME: Should be Disabled, because other tables have to be fixed up first -- Actually seems to work now, let's test it a bit more
     local usedfonts = {}
     for i=1,#usedcids do
       local cid = usedcids[i][1]
@@ -527,14 +523,13 @@ function myfunc(buf, i0, fontid, usedcids, encoding, trust_widths)
     for i=1,#glyphs do
       glyphs[i].cidfont = usedfonts[glyphs[i].cidfont]
     end
-    -- TODO: CIDFont / Privates subsetting... DONE(?)
-    -- TODO: Subrs subsetting... Instead of deleting unused SubRs, we only make them empty.
-    --       This avoids problems with renumberings whiuch would have to be consitant across
-    --       Fonts in some odd way, because they might be used by globalsubrs.
+    -- Subrs subsetting... Instead of deleting unused SubRS, we only make them empty.
+    -- This avoids problems with renumberings which would have to be consitant across
+    -- Fonts in some odd way, because they might be used by globalsubrs.
     for i=1,#glyphs do
       local g = glyphs[i]
       local private = top.Privates[g.cidfont or 1]
-      local parsed = parse_charstring(g.cs, top.GlobalSubrs, private.Subrs) -- TODO: Implement
+      local parsed = parse_charstring(g.cs, top.GlobalSubrs, private.Subrs)
       local width = parsed[1][2]
       if width then
         width = width + (private.nominalWidthX or 0)
@@ -576,15 +571,11 @@ function myfunc(buf, i0, fontid, usedcids, encoding, trust_widths)
   end
   return require'luametalatex-font-cff'(top), bbox
 end
--- local file = io.open(arg[1])
--- local buf = file:read'a'
--- file:close()
--- io.open(arg[3], 'w'):write(myfunc(buf, 1, 1, nil, {{3}, {200}, {1000}, {1329}, {1330}, {1331}})):close()
+
 return function(filename, fontid, encoding) return function(fontdir, usedcids)
-  local file = io.open(filename)
-  local buf = file:read'a'
+  local file <close> = readfile('opentype', filename)
+  local buf = file()
   local i = 1
-  file:close()
   local magic = buf:sub(1, 4)
   if magic == "ttcf" or magic == "OTTO" then
     -- assert(not encoding) -- nil or false
@@ -598,4 +589,3 @@ return function(filename, fontid, encoding) return function(fontdir, usedcids)
   fontdir.bbox = bbox
   return content
 end end
--- io.open(arg[3], 'w'):write(myfunc(buf, 1, 1, require'parseEnc'(arg[2]), {{string.byte'a'}, {string.byte'b'}, {string.byte'-'}})):close()

luametalatex-pdf-font-map.lua

@@ -1,3 +1,5 @@
+local readfile = require'luametalatex-readfile'
+
 local purenumber = lpeg.R'09'^1
 local optoperator = lpeg.C(lpeg.S'+-='^-1)*lpeg.C(lpeg.P(1)^0)
 local commentchar = lpeg.S' %*;#'+-1
@@ -38,9 +40,8 @@ local function mapfile(filename, operator)
   if not operator then
     operator, filename = optoperator:match(filename)
   end
-  local file = io.open(kpse.find_file(filename, 'map'))
+  local file <close> = readfile('map', filename)
   for line in file:lines() do mapline(line, operator) end
-  file:close()
 end
 local function reset()
   for k in next, fontmap do

luametalatex-pdf-font-node.lua

@@ -0,0 +1,42 @@
+local writer = require'luametalatex-nodewriter'
+local utils = require'luametalatex-pdf-utils'
+local to_bp, strip_floats = utils.to_bp, utils.strip_floats
+local prepared = {}
+return {
+  buildfont = function(pdf, fontdir, usedcids)
+    local designsize = fontdir.designsize
+    local scale = 1/to_bp(designsize)
+    local bbox = {0, 0, 0, 0}
+    local matrix = {scale, 0, 0, scale, 0, 0}
+    local widths = {}
+    local first_cid = usedcids[1][1]-1
+    local charprocs = {}
+    local prev = 0
+    local characters = fontdir.characters
+    local prepared = assert(prepared[fontdir])
+    for i=1,#usedcids do
+      local used = usedcids[i]
+      local glyph = characters[used[1]]
+      for j=prev+1,used[1]-first_cid-1 do
+        widths[j] = 0
+      end
+      prev = used[1]-first_cid
+      widths[prev] = to_bp(glyph.width)
+      charprocs[i] = string.format("/G%i %i 0 R", used[1], prepared[used[1]])
+    end
+    return bbox, matrix, pdf:indirect(nil, strip_floats('[' .. table.concat(widths, ' ') .. ']')), '<<' .. table.concat(charprocs) .. '>>'
+  end,
+  prepare = function(fontdir, usedglyphs, pdf, fontdirs, allusedglyphs)
+    local state = prepared[fontdir]
+    if not state then
+      state = {}
+      prepared[fontdir] = state
+    end
+    for gid in next, usedglyphs do if tonumber(gid) and not state[gid] then
+        local stream, annots
+        stream, state.resources, annots = writer(pdf, fontdir.characters[gid].node, fontdirs, allusedglyphs, nil, state.resources)
+        state[gid] = pdf:stream(nil, '', stream)
+        assert(annots == '')
+    end end
+  end,
+}

luametalatex-pdf-font-pk.lua

@@ -1,6 +1,7 @@
 local pk_global_resolution, pk_resolution_is_fixed
 local pdfvariable = pdf.variable
 
+local readfile = require'luametalatex-readfile'
 local read_pk = require'luametalatex-font-pk'
 local strip_floats = require'luametalatex-pdf-utils'.strip_floats
 return function(pdf, fontdir, usedcids)
@@ -13,7 +14,8 @@ return function(pdf, fontdir, usedcids)
     pk_resolution_is_fixed = pdfvariable.pkfixeddpi ~= 0
     kpse.init_prog("LUATEX", pk_global_resolution, pkmode ~= '' and pkmode or nil, nil) -- ?
   end
-  local pk = read_pk(kpse.find_file(fontdir.name, 'pk', pk_resolution_is_fixed and pk_global_resolution or (pk_global_resolution*fontdir.size/fontdir.designsize+.5)//1))
+  local f <close> = assert(readfile('pk', fontdir.name, pk_resolution_is_fixed and pk_global_resolution or (pk_global_resolution*fontdir.size/fontdir.designsize+.5)//1))
+  local pk = read_pk(f())
   local designsize = pk.designsize/1044654.326 -- 1044654.326=2^20*72/72.27 -- designsize in bp
   local hscale = 65536/pk.hppp / designsize -- 65291.158=2^16*72/72.27
   local vscale = 65536/pk.vppp / designsize -- 65291.158=2^16*72/72.27

luametalatex-pdf-font-t1.lua

@@ -1,12 +1,13 @@
--- nodefont = true
+local readfile = require'luametalatex-readfile'
+
 -- Some helpers:
--- A kpse wrapper
 local serialize_cff = require'luametalatex-font-cff'
 local serializet2 = require'luametalatex-font-t2'
 local parseT1 = require'luametalatex-font-t1'
 local t1tot2 = require'luametalatex-font-t1tot2'
 return function(filename, reencode)
-  local parsed_t1 = parseT1(filename)
+  local file <close> = readfile('type1', filename)
+  local parsed_t1 = parseT1(file())
   return function(f, usedcids)
     f.bbox = parsed_t1.FontBBox
     local fonttable = {
@@ -39,7 +40,7 @@ return function(filename, reencode)
       -- LanguageGroup = parsed_t1.Private.LanguageGroup,
     }
     if not reencode and parsed_t1.Encoding == "StandardEncoding" then
-      reencode = kpse.find_file("8a.enc", "enc files")
+      reencode = '8a'
     end
     if reencode then
       parsed_t1.Encoding = require'luametalatex-font-enc'(reencode)

luametalatex-pdf-font-ttf.lua

@@ -1,5 +1,7 @@
-local sfnt = require'libSfnt'
-local stdnames = require'ttfstaticstrings'
+local readfile = require'luametalatex-readfile'
+
+local sfnt = require'luametalatex-font-sfnt'
+local stdnames = require'luametalatex-font-ttf-data'
 local function round(x)
   local i, f = math.modf(x)
   if f < 0 then
@@ -9,9 +11,8 @@ local function round(x)
   end
 end
 local function addglyph(glyph, usedcids, cidtogid)
-  -- FIXME: Pseudocode
   if string.unpack(">i2", glyph) < 0 then -- We have a composite glyph.
-    -- This is an untested mess. Disaster will follow.
+    -- This is a mess. Disaster will follow.
     local offset = 11
     while offset do
       local flags, component = string.unpack(">I2I2", glyph, offset)
@@ -21,7 +22,7 @@ local function addglyph(glyph, usedcids, cidtogid)
         usedcids[gid] = {component}
         cidtogid[component] = gid
       end
-      glyph = glyph:sub(1, offset-1) .. string.pack(">I2", gid).. glyph:sub(offset+2)
+      glyph = glyph:sub(1, offset+1) .. string.pack(">I2", gid-1).. glyph:sub(offset+4)
       offset = flags&32==32 and offset + 4 + (flags&1==1 and 4 or 2) + (flags&8==8 and 2 or (flags&64==64 and 4 or (flags&128==128 and 8 or 0)))
     end
   end
@@ -68,19 +69,10 @@ local function readpostnames(buf, i, usedcids, encoding)
   return newusedcids
 end
 return function(filename, fontid, reencode)
-  local file = io.open(filename)
-  local buf = file:read'a'
-  file:close()
+  local file <close> = assert(readfile('truetype', filename))
+  local buf = file()
   local magic, tables = sfnt.parse(buf, 1, fontid)
-  if magic ~=  "\0\1\0\0" then error[[Invalid TTF font]] end
-  -- TODO: Parse post table and add reencoding support
-  -- if tables.post and string.unpack(">I4", buf, tables.post[1]) == 0x00020000 and reencode then
-  --   local encoding = require'parseEnc'(reencode)
-  --   if encoding then
-  --     local names = {}
-  --     local off = tables.post[1] + 4
-  --     for i = 1,string.unpack(">I2", buf, tables.maxp[1] + 4) do
-
+  if magic ~= "\0\1\0\0" then error[[Invalid TTF font]] end
   return function(fontdir, usedcids)
     if reencode and string.unpack(">I4", buf, tables.post[1]) == 0x00020000 then
       usedcids = readpostnames(buf, tables.post[1] + 32, usedcids, require'luametalatex-font-enc'(reencode))

luametalatex-pdf-font.lua

@@ -1,5 +1,9 @@
 local strip_floats = require'luametalatex-pdf-utils'.strip_floats
 
+local buildtff = require'luametalatex-pdf-font-ttf'
+local buildcff = require'luametalatex-pdf-font-cff'
+local buildcff_from_t1 = require'luametalatex-pdf-font-t1'
+
 local tounicode = {
   [-3] = require'luametalatex-pdf-font-cmap3',
          require'luametalatex-pdf-font-cmap1',
@@ -181,9 +185,9 @@ local function buildfont0cff(pdf, fontdir, usedcids)
   else
     if fontdir.filename then
       if fontdir.format == "type1" then
-        cff = require'luametalatex-pdf-font-t1'(fontdir.filename, fontdir.encoding)(fontdir, usedcids)
+        cff = buildcff_from_t1(fontdir.filename, fontdir.encoding)(fontdir, usedcids)
       elseif fontdir.format == "opentype" then
-        cff = require'luametalatex-pdf-font-cff'(fontdir.filename, 1, fontdir.encodingbytes == 1 and (fontdir.encoding or true))(fontdir, usedcids)
+        cff = buildcff(fontdir.filename, 1, fontdir.encodingbytes == 1 and (fontdir.encoding or true))(fontdir, usedcids)
       else
         error[[Unsupported format]]
       end
@@ -207,7 +211,7 @@ local function buildfont0ttf(pdf, fontdir, usedcids)
   if fontdir.ttf then
     ttf = fontdir:ttf(usedcids) -- WARNING: If you implement this: You always have to add a .notdef glyph at index 0. This one is *not* included in usedcids
   else
-    ttf = require'luametalatex-pdf-font-ttf'(fontdir.filename, 1, fontdir.encoding)(fontdir, usedcids)
+    ttf = buildtff(fontdir.filename, 1, fontdir.encoding)(fontdir, usedcids)
   end
   local lastcid = -1
   local cidtogid = {}
@@ -234,8 +238,7 @@ local function buildfont0(pdf, fontdir, usedcids)
   local enc
   if fontdir.encodingbytes == 1 then
     enc = cidmap1byte(pdf)
-  elseif false then -- FIXME: This should only be used for encodingbyzes == -3 (variable, max 3)
-    fontdir.encodingbytes = -3 -- FIXME
+  elseif fontdir.encodingbytes == -3 then -- (variable, max 3)
     enc = cidmap3byte(pdf)
   else
     enc = "/Identity-H"

luametalatex-pdf-image-pdf.lua

@@ -31,8 +31,16 @@ end
 
 local pdf_functions = {}
 
-local function open_pdfe(img)
-  local file = pdfe.open(img.filepath)
+local function open_pdfe(img, f)
+  local file
+  if f and f.file then
+    file = pdfe.openfile(f.file)
+    f.file = nil
+  elseif img.filedata then
+    file = pdfe.new(img.filedata, #img.filedata)
+  elseif img.filepath then
+    file = pdfe.open(img.filepath)
+  end
   do
     local userpassword = img.userpassword
     local ownerpassword = img.ownerpassword
@@ -51,8 +59,8 @@ local function open_pdfe(img)
     assert(false)
   end
 end
-function pdf_functions.scan(img)
-  local file = open_pdfe(img)
+function pdf_functions.scan(img, f)
+  local file = open_pdfe(img, f)
   img.pages = pdfe.getnofpages(file)
   img.page = img.page or 1
   if img.page > img.pages then

luametalatex-pdf-image-png.lua

@@ -6,17 +6,18 @@ local parse = setmetatable({
   -- PLTE = below,
   -- IDAT = below,
   -- IEND = below,
-  -- I'm not yet sure what to do about the following four color management chunks:
-  -- These two will probably be ignored (if you care about this stuff, you probably
-  -- prefer an ICC profile anyway. Also especially cHRM requires some weird computations.)
-  -- cHRM = TODO, -- ignore?
-  -- gAMA = TODO, -- ignore?
+  -- I originally thought about ignoring cHRM and gAMA (if you care about this stuff, you probably
+  -- prefer an ICC profile anyway) but it was interesting to think about so it got implemented.
+  -- But, gAMA is ONLY supported in combination with cHRM and not on it's own.
+  -- cHRM = below,
+  -- gAMA = below,
   -- iCCP is implemented, but profiles are not cached, so it might include the
   -- same profile many times
   -- iCCP = below,
   -- I would expect sRGB to be the most common, but it is a bit complicated because
-  -- PDF seems to require us to ship an actual ICC profile to support sRGB. Maybe later.
-  -- sRGB = TODO,
+  -- PDF seems to require us to ship an actual ICC profile to support sRGB. Luckily,
+  -- such a profile is part of TeXLive anyway.
+  -- sRGB = below,
   sBIT = ignore,
   bKGD = ignore, -- Background color. Ignored since we support transparency
   hIST = ignore, -- Color histogram
@@ -203,13 +204,8 @@ end
 
 local png_functions = {}
 
-function png_functions.scan(img)
-  local file = io.open(img.filepath)
-  if not file then
-    error[[PDF image could not be opened.]]
-  end
-  local buf = file:read'a'
-  file:close()
+function png_functions.scan(img, f)
+  local buf = f()
   local t = run(buf, 1, #buf, 'IDAT')
   img.pages = 1
   img.page = 1
@@ -228,10 +224,7 @@ local intents = {[0]=
 }
 local function srgb_lookup(pfile, intent)
   if not srgb_colorspace then
-    local f = io.open(kpse.find_file'sRGB.icc.zlib')
-    local profile = f:read'a'
-    f:close()
-    local objnum = pfile:stream(nil, '/Filter/FlateDecode/N ' .. tostring(colortype & 2 == 2 and '3' or '1'), t.iCCP, nil, true)
+    local objnum = pfile:stream(nil, '/N 3', 'sRGB.icc', true)
     srgb_colorspace = string.format('[/ICCBased %i 0 R]', objnum)
   end
   return objnum, intents[intent] or ''
@@ -259,12 +252,11 @@ local function rawimage(t, content)
 end
 
 function png_functions.write(pfile, img)
-  local file = io.open(img.filepath)
-  if not file then
-    error[[PDF image could not be opened.]]
+  local buf = img.filedata
+  if not buf then
+    local f <close> = assert(io.open(img.filepath, 'rb'))
+    buf = f:read'a'
   end
-  local buf = file:read'a'
-  file:close()
   local t = run(buf, 1, #buf, 'IEND')
   local colorspace
   local intent = ''

luametalatex-pdf-image.lua

@@ -1,3 +1,5 @@
+local readfile = require'luametalatex-readfile'
+
 local rawset = rawset
 local setdata = node.direct.setdata
 local nodenew = node.direct.new
@@ -61,14 +63,18 @@ local function scan(img)
     if m ~= meta then img = new(img) end
     real = real_images[img]
     if real.stream then error[[stream images are not yet supported]] end
-    assert(real.filename)
     -- TODO: At some point we should just take the lowercased extension
     local imagetype = real.filename:match'%.pdf$' and 'pdf'
                    or real.filename:match'%.png$' and 'png'
                    or error'Unsupported image format'
-    real.filepath = assert(kpse.find_file(real.filename), "Image not found")
     real.imagetype = imagetype
-    imagetypes[imagetype].scan(real)
+    local f <close>, path = assert(readfile('image', real.filename))
+    if f.file then
+      real.filepath = path
+    else
+      real.filedata = f.data
+    end
+    imagetypes[imagetype].scan(real, f)
     setmetatable(img, restricted_meta)
   end
   img.transform = img.transform or 0

luametalatex-pdf-pagetree.lua

@@ -1,12 +1,13 @@
+local pdf = pdf
 local min = math.min
 local format = string.format
 local concat = table.concat
 local pdfvariable = pdf.variable
-local function write(pdf, tree, total, max)
-  tree = tree or pdf.pages
+local function write(pfile, tree, total, max)
+  tree = tree or pfile.pages
   if #tree == 0 then 
-    local id = pdf:getobj()
-    pdf:indirect(id, '<</Type/Pages/Kids[]/Count 0>>')
+    local id = pfile:getobj()
+    pfile:indirect(id, '<</Type/Pages/Kids[]/Count 0>>')
     return id
   end
   max = max or 6 -- These defaults only work on the lowest level
@@ -20,37 +21,37 @@ local function write(pdf, tree, total, max)
     local id = tree[-i]
     newtree[i+1] = id
     if 0 == i % 6 and #tree > 6 then
-      local parentid = pdf:getobj()
+      local parentid = pfile:getobj()
       newtree[-(i//6)] = parentid
       parent = format("/Parent %i 0 R", parentid)
     elseif #tree <= 6 then
-      parent = pdfvariable.pagesattr
+      parent = pdfvariable.pagesattr .. pdf.pagesattributes
     end
-    pdf:indirect(id, format('<</Type/Pages%s/Kids[%s 0 R]/Count %i>>', parent, concat(tree, ' 0 R ', 6*i+1, min(#tree, 6*i+6)), min(remaining, max)))
+    pfile:indirect(id, format('<</Type/Pages%s/Kids[%s 0 R]/Count %i>>', parent, concat(tree, ' 0 R ', 6*i+1, min(#tree, 6*i+6)), min(remaining, max)))
     remaining = remaining - max
   end
   if newtree[0] then
-    return write(pdf, newtree, total, max*6)
+    return write(pfile, newtree, total, max*6)
   end
   return newtree[1]
 end
-local function newpage(pdf)
-  local pages = pdf.pages
+local function newpage(pfile)
+  local pages = pfile.pages
   local pagenumber = #pages+1
   local pageid = pages.reserved and pages.reserved[pagenumber]
   if pageid then
     pages.reserved[pagenumber] = nil
   else
-    pageid = pdf:getobj()
+    pageid = pfile:getobj()
   end
   pages[pagenumber] = pageid
   if 1 == pagenumber % 6 then
-    pages[-((pagenumber-1)//6)] = pdf:getobj()
+    pages[-((pagenumber-1)//6)] = pfile:getobj()
   end
   return pageid, pages[-((pagenumber-1)//6)]
 end
-local function reservepage(pdf, num)
-  local pages = pdf.pages
+local function reservepage(pfile, num)
+  local pages = pfile.pages
   if pages[num] then return pages[num] end
   local reserved = pages.reserved
   if reserved then
@@ -59,7 +60,7 @@ local function reservepage(pdf, num)
     reserved = {}
     pages.reserved = reserved
   end
-  reserved[num] = pdf:getobj()
+  reserved[num] = pfile:getobj()
   return reserved[num]
 end
 return {

luametalatex-pdf-resources.lua

@@ -0,0 +1,37 @@
+local format = string.format
+local concat = table.concat
+local next = next
+
+local temp_table = {}
+local resources_meta = {
+  __index = function(t, k)
+    local v = {}
+    t[k] = v
+    return v
+  end,
+  __call = function(t, additional)
+    local temp_table = temp_table
+    local i=1
+    local next_init = '<</%s<<'
+    for kind, entries in next, t do
+      temp_table[i] = format(next_init, kind)
+      next_init = '>>/%s<<'
+      i = i+1
+      for name, entry in next, entries do
+        temp_table[i] = format('/%s %i 0 R', name, entry)
+        i = i+1
+      end
+    end
+    if i == 1 then return format('<<%s>>', additional or '') end
+    temp_table[i] = format('>>%s>>', additional or '')
+    local result = concat(temp_table)
+    for j=1,i do
+      temp_table[j] = nil
+    end
+    return result
+  end,
+}
+
+return function(t)
+  return setmetatable(t or {}, resources_meta)
+end

luametalatex-pdf.lua

@@ -1,3 +1,5 @@
+local readfile = require'luametalatex-readfile'
+
 local format = string.format
 local byte = string.byte
 local pack = string.pack
@@ -8,6 +10,7 @@ local assigned = {}
 local delayed = {}
 local compress = xzip.compress
 local pdfvariable = pdf.variable
+local digest = sha2.digest256
 -- slightly tricky interface: No/nil return means that the objects content
 -- isn't known yet, while false indicates a delayed object.
 local function written(pdf, num)
@@ -15,7 +18,7 @@ local function written(pdf, num)
   if not num or num == assigned then return end
   return num ~= delayed
 end
--- raw: Pass on preencoded stream. Currently ignored.
+-- raw: Pass on preencoded stream.
 local function stream(pdf, num, dict, content, isfile, raw)
   if not num then num = pdf:getobj() end
   if pdf[num] ~= assigned then
@@ -23,9 +26,8 @@ local function stream(pdf, num, dict, content, isfile, raw)
   end
   pdf[num] = {offset = pdf.file:seek()}
   if isfile then
-    local f = io.open(content)
-    content = f:read'a'
-    f:close()
+    local file <close> = readfile('pdf_stream', content, nil)
+    content = file()
   end
   local level = not raw and pdfvariable.compresslevel or 0
   local filter = ''
@@ -56,9 +58,8 @@ local function indirect(pdf, num, content, isfile, objstream)
     error[[Invalid object]]
   end
   if isfile then
-    local f = io.open(content)
-    content = f:read'a'
-    f:close()
+    local file <close> = readfile('pdf_dict', content, nil)
+    content = file()
   end
   if objstream ~= false and pdfvariable.objcompresslevel ~= 0 then
     objstream = objstream or true
@@ -165,8 +166,12 @@ local pdfmeta = {
 }
 pdfmeta.__index = pdfmeta
 local function open(filename)
-  local file = io.open(filename, 'w')
-  file:write"%PDF-X.X\n%🖋\n"
+  local file, msg = io.open(filename, 'wb')
+  if not file then
+    tex.error('Unable to open output file', string.format("Opening the output file %q failed. According to your system, the reason is: %q. If you continue, all output will be discarded.", filename, msg))
+    file = assert(io.tmpfile())
+  end
+  file:write"%PDF-X.X\n%\xC1\xAC\xC1\xB4\n"
   return setmetatable({file = file, version = '1.7', [0] = 0, pages = {}, objstream = {}}, pdfmeta)
 end
 return {

luametalatex-readfile.lua

@@ -0,0 +1,89 @@
+local io_open = io.open
+local write = texio.write
+local find_file = kpse.find_file
+
+local callbacks = require'luametalatex-callbacks'
+
+-- local categories = { data = 1, map = 2, image = 3, subset = 4, font = 5 -- , enc = 6, pdf_stream = 7, pdf_stream = 8, silent = 9}
+
+local our_callbacks = {
+  vf       = {'vf',             false, 'rb', 'find_vf_file',       'read_vf_file'},
+  tfm      = {'tfm',            false, 'rb', 'find_font_file',     'read_font_file'},
+  map      = {'map',            2,     'r',  'find_map_file',      'read_map_file'},
+  enc      = {'enc files',      false, 'r',  'find_enc_file',      'read_enc_file'},
+  type1    = {'type1 fonts',    4,     'rb', 'find_type1_file',    'read_type1_file'},
+  truetype = {'truetype fonts', 4,     'rb', 'find_truetype_file', 'read_truetype_file'},
+  opentype = {'opentype fonts', 4,     'rb', 'find_opentype_file', 'read_opentype_file'},
+  pk       = {'pk',             4,     'rb', 'find_pk_file',       'read_pk_file'},
+  image    = {'tex',            3,     'rb', 'find_image_file',    'read_image_file'},
+  data     = {'tex',            1,     'rb', 'find_data_file',     'read_data_file'},
+}
+
+local start_categories = { [0] = '?', '(', '{', '<', '<', '<<' }
+local stop_categories  = { [0] = '?', ')', '}', '>', '>', '>>' }
+
+local function stop_file(t)
+  local cb = callbacks.stop_file
+  if cb then
+    cb(t.category)
+  else
+    write(stop_categories[t.category] or '')
+  end
+  if t.file then t.file:close() end
+end
+
+local meta_file = {
+  __close = stop_file,
+  __call = function(t) return t.file:read'a' end,
+  close = stop_file,
+  lines = function(t, ...) return t.file:lines(...) end,
+}
+meta_file.__index = meta_file
+
+local meta_data = {
+  __close = stop_file,
+  __call = function(t) return t.data end,
+  close = stop_file,
+  lines = function(t, ...) return t.data:gmatch('([^\n]*)\n') end,
+}
+meta_data.__index = meta_data
+
+return function(kind, name, ...)
+  local handle
+  local kind_info = our_callbacks[kind]
+  local msg
+  if kind_info then
+    local find_callback = callbacks[kind_info[4]]
+    if find_callback then
+      name, msg = find_callback(name, ...)
+    else
+      name, msg = find_file(name, kind_info[1], ...)
+    end
+    if not name then return name, msg end
+    handle = {category = kind_info[2]}
+    local read_callback = callbacks[kind_info[5]]
+    if read_callback then
+      local success, data, size = read_callback(name)
+      if not success then return success, data end
+      if size < #data then data = data:sub(1,size) end
+      handle.data = data, data
+      setmetatable(handle, meta_data)
+    else
+      local f f, msg = io_open(name, kind_info[3])
+      if not f then return f, msg end
+      handle.file = f
+      setmetatable(handle, meta_file)
+    end
+    if handle.category then
+      local cb = callbacks.start_file
+      if cb then
+        cb(handle.category, name)
+      else
+        write(start_categories[handle.category] .. name)
+      end
+    end
+  else
+    error[[Unknown file]]
+  end
+  return handle, name
+end

luametalatex.ini

@@ -1,14 +1,19 @@
-% Thomas Esser, 1998. public domain.
-%
+% Based on lualatex.ini, originally written 2008 by Karl Berry. Public domain.
+
+\input luametatexconfig.tex
 
-\scrollmode
 \begingroup
-\catcode`\{=1
-\catcode`\}=2
-\global\chardef\eTeXversion=2
-\global\def\eTeXrevision{.2}
+  \catcode`\{=1 %
+  \catcode`\}=2 %
+  % Set up job name quoting before latex.ltx
+  % Web2c pdfTeX/XeTeX quote job names containing spaces, but LuaTeX does
+  % not do this at the engine level. The behaviour can be changed using
+  % a callback. Originally this code was loaded via lualatexquotejobname.tex
+  % but that required a hack around latex.ltx: the behaviour has been altered
+  % to allow the callback route to be used directly.
+  \global\everyjob{\directlua{require("lualatexquotejobname.lua")}}
+  \global\chardef\eTeXversion=2
+  \global\def\eTeXrevision{.2}
 \endgroup
-% \let\savedversionofdump\dump
-% \let\dump\relax
+
 \input latex.ltx
-\endinput

luametalatex.lua

@@ -1,122 +1,75 @@
--- Some helpers based on Penlight
-local absdir, dirsep
-do
-  local sep = package.config:sub(1,1)
-  local is_windows = sep == "\\"
-  dirsep = lpeg.S(is_windows and '\\/' or '/')
-  local anchor_pattern = lpeg.Cs(is_windows
-    and lpeg.P'\\\\' + dirsep/'\\' + 1*lpeg.P':'*dirsep^-1/'\\'
-    or  lpeg.P'//' + dirsep^1/'/')
-  function isabs(P)
-    return P:sub(1,1) == '/' or (is_windows and (P:sub(1,1)=='\\' or P:sub(2,2)==':'))
-  end
-  local insert, remove, concat = table.insert, table.remove, table.concat
-  function normpath(P)
-    -- Split path into anchor and relative path.
-    local anchor, P = ((anchor_pattern + lpeg.Cc'') * lpeg.C(lpeg.P(1)^0)):match(P)
-    if is_windows then
-      P = P:gsub('/','\\')
-    end
-    local parts = {}
-    for part in P:gmatch('[^'..sep..']+') do
-      if part == '..' then
-        if #parts ~= 0 and parts[#parts] ~= '..' then
-          remove(parts)
-        else
-          insert(parts, part)
-        end
-      elseif part ~= '.' then
-        insert(parts, part)
-      end
-    end
-    P = anchor..concat(parts, sep)
-    if P == '' then P = '.' end
-    return P
-  end
-  function join(p1,p2,...)
-    if select('#',...) > 0 then
-      local p = join(p1,p2)
-      return join(p, ...)
-    end
-    if isabs(p2) then return p2 end
-    local endc = p1:sub(#p1,#p1)
-    if endc ~= "/" and (not is_windows or endc ~= "\\") and endc ~= "" then
-      p1 = p1..sep
-    end
-    return p1..p2
-  end
-  function absdir(P,pwd)
-    local use_pwd = pwd ~= nil
-    pwd = pwd or lfs.currentdir()
-    if not isabs(P) then
-      P = join(pwd,P)
-    elseif is_windows and not use_pwd and P:sub(2,2) ~= ':' and P:sub(2,2) ~= '\\' then
-      P = pwd:sub(1,2)..P -- attach current drive to path like '\\fred.txt'
-    end
-    return normpath(P) .. sep
-  end
-end
--- Who are we anyway?
 local format = os.selfname -- or os.selfcore, I couldn't find a difference yet
 local ourname = arg[0] -- This might not be os.selfarg[0]
+-- We want to drop arg[0] aka. ourname from the arguments to avoid recursion
 if os.selfarg[0] == ourname then
   ourname = nil
 end
-for i, a in ipairs(os.selfarg) do
+local run
+
+local arg_pattern, late_arg_pattern do
+  local l = lpeg or require'lpeg'
+  -- We intepret two arguments directly
+  local early_args = 'run' * l.Cmt(l.P'=' + -1, function() run = true return true end)
+                   + 'progname=' * l.Cmt(0, function(s, off) format = s:sub(off) return true end)
   -- LuaMetaTeX needs -- to introduce parameters,
   -- but fmtutil uses just -. Let's rewrite this on the fly:
+  local maybe_option = ('-' * ('-' + l.Cc'-') * #(early_args^-1))^-1
+  local quote = l.Cc(os.type == 'windows' and '"' or "'")
+  local escape
+  if os.type == 'windows' then
+    -- Windows: " -> "^"" The ^ is for cmd escaping, the """ is for command line splitting escaping,
+    -- backslashes still have to be escaped, but only in front of " or \
+    -- Additionally, "%" always as to be escaped for some variable substitution
+    -- pass before everything else.
+    -- WARNING: This works with luametatex's argument splitting, but don't generally rely
+    -- on it for other Windows programs. There are two standard Windows ways which are incompatible...
+    escape = '\\' * #l.S'\\"' * l.Cc'\\' + '"' * l.Cc'^""' + '%' * l.Cc'%' + 1
+  else
+    -- POSIX: We escape with single quotes, so only single quotes need escaping
+    escape = "'" * l.Cc"\\''" + 1
+  end
+  arg_pattern = l.Cs(l.Cc' ' * maybe_option * quote * escape^0 * quote)
+  late_arg_pattern = l.Cs(l.Cc' ' * quote * escape^0 * quote)
+end
+
+for i, a in ipairs(os.selfarg) do
   if a == ourname then -- Avoid recursion
     table.remove(os.selfarg, i)
     ourname = nil
     a = os.selfarg[i]
   end
-  if a == "--" then break end
-  a = a:gsub("^%-%-?", "--")
-  os.selfarg[i] = a
-  if a:sub(1, 11) == "--progname=" then
-    format = a:sub(12)
-  elseif a == '--ini' then
-    is_initex = true
-  end
+  if a == "--" then arg_pattern = late_arg_pattern end -- This convention is not respected by luametatex itself
+  os.selfarg[i] = arg_pattern:match(a)
 end
-local dir = absdir(os.selfdir)
-local dirseparators = {((lpeg.S'\\/'^1 + 1 * lpeg.P':' * lpeg.S'\\/'^-1) * lpeg.Cp() * ((1-lpeg.S'\\/')^0*lpeg.S'\\/'*lpeg.Cp())^0):match(dir)}
--- First step: Find our actual format.
-local init_script = format .. "-init.lua"
-local texmf_dir = "tex/" .. format .. "/base/" .. init_script
-local paths = {
-  init_script,
-  "share/texmf-local/" .. texmf_dir,
-  "share/texmf-dist/" .. texmf_dir,
-  "share/texmf/" .. texmf_dir,
-  "texmf-local/" .. texmf_dir,
-  "texmf-dist/" .. texmf_dir,
-  "texmf/" .. texmf_dir,
-}
-for i = #dirseparators, 1, -1 do
-  dir = dir:sub(1, dirseparators[i] - 1)
-  for _, subdir in ipairs(paths) do
-    local full_path = dir .. subdir
-    local attr = lfs.attributes(full_path)
-    if attr then
-      dir = full_path
-      goto FOUND
+os.setenv("engine", status.luatex_engine)
+
+local kpse_call = io.popen(string.format("kpsewhich -progname%s -format lua -all -must-exist%s-init.lua", late_arg_pattern:match(format), late_arg_pattern:match(format)))
+local file
+repeat
+  file = kpse_call:read()
+until not assert(file, "Unable to find initialization script. Aborting."):match('^%.')
+
+if not kpse_call:close() then
+  error(file)
+end
+
+local geterrorcode
+if os.type == 'windows' then
+  function geterrorcode(ec) return ec end
+else
+  function geterrorcode(ec) return ec & 0xFF == 0 and ec >> 8 or 0xFF end
+end
+
+local firstargs = string.format("luametatex --permitloadlib%s%s", late_arg_pattern:match('--lua=' .. file), late_arg_pattern:match('--arg0=' .. os.selfarg[0]))
+local args = table.concat(os.selfarg)
+if run then -- The user wants to take care of everything
+  os.exit(geterrorcode(os.execute(firstargs .. args)))
+else
+  for i = 1, 5 do
+    local status = geterrorcode(os.execute(string.format("%s --run=%i%s", firstargs, i, args)))
+    if status ~= 75 then
+      os.exit(status)
     end
   end
+  os.exit(75)
 end
-error[[CRITICAL: Initialization script not found]]
-::FOUND::
--- table.insert(arg, 1, "--lua=" .. dir)
--- table.insert(arg, 1, "luametatex")
--- arg[0] = nil
--- local _, msg = os.exec(arg)
--- error(msg)
-os.setenv("engine", status.luatex_engine)
-local ret_value
-local args = os.selfarg[1] and " \"" .. table.concat(os.selfarg, "\" \"") .. "\"" or ""
-if is_initex then
-  ret_value = os.execute(string.format("luametatex \"--lua=%s\" --arg0=\"%s\"%s", dir, os.selfarg[0], args))
-else
-  ret_value = os.execute(string.format("luametatex \"--fmt=%s\" \"--lua=%s\" --arg0=\"%s\"%s", format, dir, os.selfarg[0], args))
-end
-os.exit(x)

luametatexconfig.tex

@@ -0,0 +1,41 @@
+% Adapted based on tex-ini-files 2016-04-15: luatexconfig.tex
+
+% Load shared (PDF) settings in LuaMetaTeX
+
+\begingroup
+  \catcode`\{=1 %
+  \catcode`\}=2 %
+  \catcode`\#=6 %
+  \def\list{%
+      {compresslevel}%
+      {decimaldigits}%
+      {horigin}%
+      {minorversion}%
+      {objcompresslevel}%
+      {pkresolution}%
+      {vorigin}%
+  }%
+  % LuaMetaTeX doesn't have \pdfoutput, etc.:
+  % emulate names where appropriate
+  \let\pdfoutput\outputmode
+  \let\pdfpageheight\pageheight
+  \let\pdfpagewidth\pagewidth
+  \def\do#1{%
+    \ifx\relax#1\else
+      \expandafter\edef\csname pdf#1\endcsname{\pdfvariable #1}%
+      \expandafter\do
+    \fi
+  }%
+  \expandafter\do\list\relax
+  % The file pdftexconfig.tex contains only <primitive> = <value> lines
+  % so can now be read using the (emulated) primitives
+  % This needs to be global so set \globaldefs for the rest of the group
+  \globaldefs=1 %
+  \input{pdftexconfig}%
+  % Pick up on a request for DVI mode and apply it whilst \pdfoutput is
+  % still defined
+  \ifx\dvimode\relax
+    \pdfoutput=0 %
+  \fi
+  \let\dvimode\undefined
+\endgroup