Object stream configuration option

Currently force enabled and exceptions are not specified yet

README.md

@@ -6,15 +6,20 @@ This code is in early stages of development and contains more bugs than features
 ## Prerequisites
 You need an up-to-date TeX Live installation and the latest version of LuaMetaTeX.
 
-Additionally a special library version of LuaTeX's kpathsea Lua binding is needed which is provided as a binary for Linux x64. For other platforms you might have to compile it yourself. Drop me a line if you need any instructions.
+Additionally a special library version of LuaTeX's kpathsea Lua binding is needed which is provided as a binary for Linux x64. For other platforms you might have to compile it yourself. Drop me a line if you need any instructions. (The source can be found under https://github.com/zauguin/luametalatex-kpse)
 
-## How to install
+## How to install (automatically)
+Obtain `luametatex` from ConTeXt, drop the binary into the same location where your `luatex` binary is installed and then run `install.sh`.
+
+## How to install (manually)
 Obtain `luametatex` from ConTeXt, drop the binary into the same location where your `luatex` binary is installed and copy (or sym-link) the file `luametalatex.lua` into the same directory. Additionally create a sym-link `luametalatex` to `luametatex` in the same directory. Then copy (or sym-link) this entire repo to `.../texmf-local/tex/lualatex/luametalatex`. 
 
 Finally add the line
 ```
 luametalatex luametatex language.dat,language.dat.lua --lua="$(kpsewhich luametalatex.lua)" luametalatex.ini
 ```
-to your local `fmtutil.cnf`. Then you should be able to run `fmtutil-sys --byfmt luametalatex` to generate the format.
+to your local `fmtutil.cnf` and configure paths for luametalatex in your `texmf.cnf`. Then you should be able to run `fmtutil-sys --byfmt luametalatex` to generate the format.
 
 If this worked you can built (simple) LaTeX documents using the command `luametalatex`.
+
+You can then repeat the same instructions with `luametalatex-dev` and `luametaplain` to also get access to development and plain TeX formats

install.sh

@@ -0,0 +1,45 @@
+#!/bin/sh
+ENGINE="$(which luametatex)"
+ENGINE_DIR="$(dirname "$ENGINE")"
+REPO="$(pwd)"
+cd "$(dirname "$ENGINE")"
+ln -s luametatex luametaplain
+ln -s luametatex luametalatex
+ln -s luametatex luametalatex-dev
+ln -s "$REPO/luametaplain.lua" .
+ln -s "$REPO/luametalatex.lua" .
+ln -s "$REPO/luametalatex-dev.lua" .
+while [ ! -d texmf ] && [ ! -d texmf-local ]
+do
+  LASTDIR="$(pwd)"
+  cd ..
+  if [ "$(pwd)" == "$LASTDIR" ]
+  then
+    exit 1
+  fi
+done
+if [ -d texmf ]
+then cd texmf
+else cd texmf-local
+fi
+mkdir -p tex/luameta{plain,latex{,-dev}}
+ln -s "$REPO" tex/luametaplain/base
+ln -s "$REPO" tex/luametalatex/base
+ln -s "$REPO" tex/luametalatex-dev/base
+mkdir -p web2c
+cat >> web2c/texmf.cnf << "EOF"
+TEXINPUTS.luametaplain    = $TEXMFDOTDIR;$TEXMF/tex/{luametatex,luatex,plain,generic,}//
+TEXINPUTS.luametalatex    = $TEXMFDOTDIR;$TEXMF/tex/{luametalatex,lualatex,latex,luametatex,luatex,generic,}//
+TEXINPUTS.luametalatex-dev= $TEXMFDOTDIR;$TEXMF/tex/{luametalatex,latex-dev,lualatex,latex,luametatex,luatex,generic,}//
+
+LUAINPUTS.luametaplain    = $TEXMFDOTDIR;$TEXMF/scripts/{$progname,$engine,}/{lua,}//;$TEXMF/tex/{luametatex,luatex,plain,generic,}//
+LUAINPUTS.luametalatex    = $TEXMFDOTDIR;$TEXMF/scripts/{$progname,$engine,}/{lua,}//;$TEXMF/tex/{luametalatex,lualatex,latex,luametatex,luatex,generic,}//
+LUAINPUTS.luametalatex-dev= $TEXMFDOTDIR;$TEXMF/scripts/{$progname,$engine,}/{lua,}//;$TEXMF/tex/{luametalatex,latex-dev,lualatex,latex,luametatex,luatex,generic,}//
+EOF
+
+cat >> web2c/fmtutil.cnf << "EOF"
+luametaplain luametatex language.dat,language.dat.lua --lua="$(kpsewhich luametalatex.lua)" luametaplain.ini
+luametalatex luametatex language.dat,language.dat.lua --lua="$(kpsewhich luametalatex.lua)" luametalatex.ini
+luametalatex-dev luametatex language.dat,language.dat.lua --lua="$(kpsewhich luametalatex-dev.lua)" luametalatex.ini
+EOF
+echo INSTALLED

ltexpl.ltx

@@ -1,65 +0,0 @@
-%%
-%% This is file `ltexpl.ltx',
-%% generated with the docstrip utility.
-%%
-%% The original source files were:
-%%
-%% ltexpl.dtx  (with options: `2ekernel')
-%% 
-%% This is a generated file.
-%% 
-%% The source is maintained by the LaTeX Project team and bug
-%% reports for it can be opened at https://latex-project.org/bugs.html
-%% (but please observe conditions on bug reports sent to that address!)
-%% 
-%% 
-%% Copyright (C) 1993-2019
-%% The LaTeX3 Project and any individual authors listed elsewhere
-%% in this file.
-%% 
-%% This file was generated from file(s) of the LaTeX base system.
-%% --------------------------------------------------------------
-%% 
-%% It may be distributed and/or modified under the
-%% conditions of the LaTeX Project Public License, either version 1.3c
-%% of this license or (at your option) any later version.
-%% The latest version of this license is in
-%%    https://www.latex-project.org/lppl.txt
-%% and version 1.3c or later is part of all distributions of LaTeX
-%% version 2008 or later.
-%% 
-%% This file has the LPPL maintenance status "maintained".
-%% 
-%% This file may only be distributed together with a copy of the LaTeX
-%% base system. You may however distribute the LaTeX base system without
-%% such generated files.
-%% 
-%% The list of all files belonging to the LaTeX base distribution is
-%% given in the file `manifest.txt'. See also `legal.txt' for additional
-%% information.
-%% 
-%% The list of derived (unpacked) files belonging to the distribution
-%% and covered by LPPL is defined by the unpacking scripts (with
-%% extension .ins) which are part of the distribution.
-%%% From File: ltexpl.dtx
-\input luametalatex-baseregisters
-\IfFileExists{expl3.ltx}
-  {%
-    \ifnum0%
-      \ifdefined\pdffilesize 1\fi
-      \ifdefined\filesize 1\fi
-      \ifdefined\luatexversion\ifnum\luatexversion>94 1\fi\fi
-      >0 %
-    \else
-      \message{Skipping expl3-dependent extensions}
-      \expandafter\endinput
-    \fi
-  }
-  {%
-    \message{Skipping expl3-dependent extensions}%
-    \endinput
-  }%
-\input{expl3.ltx}
-\endinput
-%%
-%% End of file `ltexpl.ltx'.

luametalatex-back-pdf.lua

@@ -1,12 +1,33 @@
 local pdf = pdf
+local pdfvariable = pdf.variable
+
 local writer = require'luametalatex-nodewriter'
 local newpdf = require'luametalatex-pdf'
 local nametree = require'luametalatex-pdf-nametree'
+local build_fontdir = require'luametalatex-pdf-font'
+local prepare_node_font = require'luametalatex-pdf-font-node'.prepare
+local fontmap = require'luametalatex-pdf-font-map'
+
+local utils = require'luametalatex-pdf-utils'
+local strip_floats = utils.strip_floats
+local to_bp = utils.to_bp
+
 local pdfname, pfile
 local fontdirs = setmetatable({}, {__index=function(t, k)t[k] = pfile:getobj() return t[k] end})
-local usedglyphs = {}
+local nodefont_meta = {}
+local usedglyphs = setmetatable({}, {__index=function(t, fid)
+  local v
+  if font.fonts[fid].format == 'type3node' then
+    v = setmetatable({generation = 0, next_generation = 0}, nodefont_meta)
+  else
+    v = {}
+  end
+  t[fid] = v
+  return v
+end})
 local dests = {}
 local cur_page
+local declare_whatsit = require'luametalatex-whatsits'.new
 local whatsit_id = node.id'whatsit'
 local whatsits = node.whatsits()
 local colorstacks = {{
@@ -25,19 +46,32 @@ local function get_pfile()
   return pfile
 end
 local outline
+local build_outline = require'luametalatex-pdf-outline'
 local function get_outline()
   if not outline then
-    outline = require'luametalatex-pdf-outline'()
+    outline = build_outline()
   end
   return outline
 end
 local properties = node.direct.properties
+local immediateassignment = token.new(5, token.command_id'convert')
+local global = token.new(0, token.command_id'prefix')
+local deadcycles = token.new(8, token.command_id'set_page_property')
+local zero_tok = token.create(0x30)
+local relax = token.new(0, 0)
+local reset_deadcycles = {
+  immediateassignment,
+  global,
+  deadcycles,
+  zero_tok,
+  relax,
+}
 token.luacmd("shipout", function()
   local pfile = get_pfile()
   local voff = node.new'kern'
-  voff.kern = tex.voffset + pdf.variable.vorigin
+  voff.kern = tex.voffset + pdfvariable.vorigin
   voff.next = token.scan_list()
-  voff.next.shift = tex.hoffset + pdf.variable.horigin
+  voff.next.shift = tex.hoffset + pdfvariable.horigin
   local list = node.direct.tonode(node.direct.vpack(node.direct.todirect(voff)))
   list.height = tex.pageheight
   list.width = tex.pagewidth
@@ -46,14 +80,16 @@ token.luacmd("shipout", function()
   local out, resources, annots = writer(pfile, list, fontdirs, usedglyphs, colorstacks)
   cur_page = nil
   local content = pfile:stream(nil, '', out)
-  pfile:indirect(page, string.format([[<</Type/Page/Parent %i 0 R/Contents %i 0 R/MediaBox[0 %i %i %i]/Resources%s%s>>]], parent, content, -math.ceil(list.depth/65781.76), math.ceil(list.width/65781.76), math.ceil(list.height/65781.76), resources, annots))
+  pfile:indirect(page, string.format([[<</Type/Page/Parent %i 0 R/Contents %i 0 R/MediaBox[0 %i %i %i]/Resources%s%s%s>>]], parent, content, -math.ceil(to_bp(list.depth)), math.ceil(to_bp(list.width)), math.ceil(to_bp(list.height)), resources(pdfvariable.pageresources), annots, pdfvariable.pageattr))
   node.flush_list(list)
-  token.put_next(token.create'immediateassignment', token.create'global', token.create'deadcycles', token.create(0x30), token.create'relax')
+  token.put_next(reset_deadcycles)
   token.scan_token()
 end, 'force', 'protected')
+
 local infodir = ""
 local namesdir = ""
 local catalogdir = ""
+local catalog_openaction
 local creationdate = os.date("D:%Y%m%d%H%M%S%z"):gsub("+0000$", "Z"):gsub("%d%d$", "'%0")
 local function write_infodir(p)
   local additional = ""
@@ -79,22 +115,44 @@ local pdf_escape = require'luametalatex-pdf-escape'
 local pdf_bytestring = pdf_escape.escape_bytes
 local pdf_text = pdf_escape.escape_text
 
+local function nodefont_newindex(t, k, v)
+  t.generation = t.next_generation
+  return rawset(t, k, v)
+end
+
 callback.register("stop_run", function()
   if not pfile then
     return
   end
+  do
+    nodefont_meta.__newindex = nodefont_newindex -- Start recording generations
+    local need_new_run = true
+    while need_new_run do
+      need_new_run = nil
+      for fid, glyphs in pairs(usedglyphs) do
+        local next_gen = glyphs.next_generation
+        if next_gen and next_gen == glyphs.generation then
+          glyphs.next_generation = next_gen+1
+          need_new_run = true
+          local f = font.getfont(fid) or font.fonts[fid]
+          prepare_node_font(f, glyphs, pfile, fontdirs, usedglyphs) -- Might become fid, glyphs
+        end
+      end
+    end
+  end
   for fid, id in pairs(fontdirs) do
-    local f = font.getfont(fid)
-    local psname = f.psname or f.fullname
+    local f = font.getfont(fid) or font.fonts[fid]
     local sorted = {}
+    local used = usedglyphs[fid]
+    used.generation, used.next_generation = nil, nil
     for k,v in pairs(usedglyphs[fid]) do
-    sorted[#sorted+1] = v
+      sorted[#sorted+1] = v
     end
     table.sort(sorted, function(a,b) return a[1] < b[1] end)
-    pfile:indirect(id, require'luametalatex-pdf-font'(pfile, f, sorted))
+    pfile:indirect(id, build_fontdir(pfile, f, sorted))
   end
   pfile.root = pfile:getobj()
-  pfile.version = string.format("%i.%i", pdf.variable.majorversion, pdf.variable.minorversion)
+  pfile.version = string.format("%i.%i", pdfvariable.majorversion, pdfvariable.minorversion)
   local destnames = {}
   for k,obj in next, dests do
     if pfile:written(obj) then
@@ -102,7 +160,7 @@ callback.register("stop_run", function()
         destnames[k] = obj .. ' 0 R'
       end
     else
-      texio.write_nl("Warning: Undefined destination %q", tostring(k))
+      texio.write_nl(string.format("Warning: Undefined destination %q", tostring(k)))
     end
   end
   if next(destnames) then
@@ -115,6 +173,9 @@ callback.register("stop_run", function()
   if outline then
     catalogdir = string.format("/Outlines %i 0 R%s", outline:write(pfile), catalogdir)
   end
+  if catalog_openaction then
+    catalogdir = catalogdir .. '/OpenAction' .. catalog_openaction
+  end
   pfile:indirect(pfile.root, string.format([[<</Type/Catalog/Version/%s/Pages %i 0 R%s>>]], pfile.version, pfile:writepages(), catalogdir))
   pfile.info = write_infodir(pfile)
   local size = pfile:close()
@@ -142,10 +203,9 @@ callback.register("stop_run", function()
   texio.write_nl(string.format("Transcript written on %s.\n", status.log_name))
 end, "Finish PDF file")
 token.luacmd("pdfvariable", function()
-  for n, t in pairs(pdf.variable_tokens) do
+  for _, n in ipairs(pdf.variable_names) do
     if token.scan_keyword(n) then
-      token.put_next(t)
-      return
+      return token.put_next(token.create('pdfvariable  ' .. n))
     end
   end
   -- The following error message gobbles the next word as a side effect.
@@ -170,9 +230,6 @@ function pdf.newcolorstack(default, mode, page)
   }
   return idx
 end
-local function sp2bp(sp)
-  return sp/65781.76
-end
 local function projected(m, x, y, w)
   w = w or 1
   return x*m[1] + y*m[3] + w*m[5], x*m[2] + y*m[4] + w*m[6]
@@ -196,19 +253,30 @@ local function get_action_attr(p, action, is_link)
     error[[FIXME]]
   elseif action_type == 1 then -- GoTo
     local id = action.id
-    if file then
-      assert(type(id) == "string")
-      action_attr = action_attr .. "/S/GoToR/D" .. pdf_bytestring(id) .. ">>"
-    else
-      local dest = dests[id]
-      if not dest then
-        dest = pfile:getobj()
-        dests[id] = dest
-      end
-      if type(id) == "string" then
-        action_attr = action_attr .. "/S/GoTo/D" .. pdf_bytestring(id) .. ">>"
+    if id then
+      if file then
+        assert(type(id) == "string")
+        action_attr = action_attr .. "/S/GoToR/D" .. pdf_bytestring(id) .. ">>"
       else
-        action_attr = string.format("%s/S/GoTo/D %i 0 R>>", action_attr, dest)
+        local dest = dests[id]
+        if not dest then
+          dest = pfile:getobj()
+          dests[id] = dest
+        end
+        if type(id) == "string" then
+          action_attr = action_attr .. "/S/GoTo/D" .. pdf_bytestring(id) .. ">>"
+        else
+          action_attr = string.format("%s/S/GoTo/D %i 0 R>>", action_attr, dest)
+        end
+      end
+    else
+      id = assert(action.page, 'GoTo action must contain either an id or a page')
+      local tokens = action.tokens
+      if file then
+        action_attr = string.format("%s/S/GoToR/D[%i %s]>>", action_attr, id-1, tokens)
+      else
+        local page_objnum = pfile:reservepage(id)
+        action_attr = string.format("%s/S/GoTo/D[%i 0 R %s]>>", action_attr, page_objnum, tokens)
       end
     end
   end
@@ -222,20 +290,21 @@ local function write_link(p, link)
   local quadStr = {}
   for i=1,#quads,8 do
     local x1, y1, x4, y4, x2, y2, x3, y3 = table.unpack(quads, i, i+7)
-    x1, y1, x2, y2, x3, y3, x4, y4 = sp2bp(x1), sp2bp(y1), sp2bp(x2), sp2bp(y2), sp2bp(x3), sp2bp(y3), sp2bp(x4), sp2bp(y4)
+    x1, y1, x2, y2, x3, y3, x4, y4 = to_bp(x1), to_bp(y1), to_bp(x2), to_bp(y2), to_bp(x3), to_bp(y3), to_bp(x4), to_bp(y4)
     quadStr[i//8+1] = string.format("%f %f %f %f %f %f %f %f", x1, y1, x2, y2, x3, y3, x4, y4)
     minX = math.min(minX, x1, x2, x3, x4)
     minY = math.min(minY, y1, y2, y3, y4)
     maxX = math.max(maxX, x1, x2, x3, x4)
     maxY = math.max(maxY, y1, y2, y3, y4)
   end
-  pfile:indirect(link.objnum, string.format("<</Type/Annot/Rect[%f %f %f %f]/QuadPoints[%s]%s>>", minX-.2, minY-.2, maxX+.2, maxY+.2, table.concat(quadStr, ' '), attr))
+  local boxes = strip_floats(string.format("/Rect[%f %f %f %f]/QuadPoints[%s]", minX-.2, minY-.2, maxX+.2, maxY+.2, table.concat(quadStr, ' ')))
+  pfile:indirect(link.objnum, string.format("<</Type/Annot%s%s>>", boxes, attr))
   for i=1,#quads do quads[i] = nil end
   link.objnum = nil
 end
 local function addlinkpoint(p, link, x, y, list, kind)
   local quads = link.quads
-  local off = pdf.variable.linkmargin
+  local off = pdfvariable.linkmargin
   x = kind == 'start' and x-off or x+off
   if link.annots and link.annots ~= p.annots then -- We started on another page, let's finish that before starting the new page
     write_link(p, link)
@@ -257,12 +326,23 @@ local function addlinkpoint(p, link, x, y, list, kind)
   end
 end
 local function linkcontext_set(linkcontext, p, x, y, list, level, kind)
+  if not p.is_page then return end
   for _,l in ipairs(linkcontext) do if l.level == level then
       addlinkpoint(p, l, x, y, list, level, kind)
   end end
 end
 
-function do_start_link(prop, p, n, x, y, outer, _, level)
+local start_link_whatsit = declare_whatsit('pdf_start_link', function(prop, p, n, x, y, outer, _, level)
+  if not prop then
+    tex.error('Invalid pdf_start_link whatsit', {"A pdf_start_link whatsit did not contain all necessary \z
+        parameters. Maybe your code hasn't been adapted to LuaMetaLaTeX yet?"})
+    return
+  end
+  if not p.is_page then
+    tex.error('pdf_start_link outside of page', {"PDF links are not allowed in Type3 charstrings or Form XObjects. \z
+        The link will be ignored"})
+    return
+  end
   local links = p.linkcontext
   if not links then
     links = {set = linkcontext_set}
@@ -271,25 +351,45 @@ function do_start_link(prop, p, n, x, y, outer, _, level)
   local link = {quads = {}, attr = prop.link_attr, action = prop.action, level = level, force_separate = false} -- force_separate should become an option
   links[#links+1] = link
   addlinkpoint(p, link, x, y, outer, 'start')
-end
-function do_end_link(prop, p, n, x, y, outer, _, level)
+end)
+local end_link_whatsit = declare_whatsit('pdf_end_link', function(prop, p, n, x, y, outer, _, level)
+  if not p.is_page then
+    tex.error('pdf_start_link outside of page', {"PDF links are not allowed in Type3 charstrings or Form XObjects. \z
+        The link will be ignored"})
+    return
+  end
   local links = p.linkcontext
-  if not links then error"No link here to end" end
+  if not links then
+    tex.error('No link here to end', {"You asked me to end a link, but currently there is no link active. \z
+        Maybe you forgot to run \\pdfextension startlink first?"})
+    return
+  end
   local link = links[#links]
+  if link.level ~= level then
+    tex.error('Inconsistent link level', {"You asked me to end a link, but the most recent link had been started at another level. \z
+        I will continue with the link for now."})
+    return
+  end
   links[#links] = nil
-  if link.level ~= level then error"Wrong link level" end
+  if not links[1] then p.linkcontext = nil end
   addlinkpoint(p, link, x, y, outer, 'final')
-end
+end)
 
-local do_setmatrix do
+local setmatrix_whatsit do
   local numberpattern = (lpeg.P'-'^-1 * lpeg.R'09'^0 * ('.' * lpeg.R'09'^0)^-1)/tonumber
   local matrixpattern = numberpattern * ' ' * numberpattern * ' ' * numberpattern * ' ' * numberpattern
-  function do_setmatrix(prop, p, n, x, y, outer)
+  setmatrix_whatsit = declare_whatsit('pdf_setmatrix', function(prop, p, n, x, y, outer)
+    if not prop then
+      tex.error('Invalid pdf_setmatrix whatsit', {"A pdf_setmatrix whatsit did not contain a matrix value. \z
+          Maybe your code hasn't been adapted to LuaMetaLaTeX yet?"})
+      return
+    end
     local m = p.matrix
     local a, b, c, d = matrixpattern:match(prop.data)
     if not a then
-      print(prop.data)
-      error[[No valid matrix found]]
+      tex.error('Invalid matrix', {"The matrix in this pdf_setmatrix whatsit does not have the expected structure and could not be parsed. \z
+          Did you provide enough parameters? The matrix needs exactly four decimal entries."})
+      return
     end
     local e, f = (1-a)*x-c*y, (1-d)*y-b*x -- Emulate that the origin is at x, y for this transformation
                                           -- (We could also first translate by (-x, -y), then apply the matrix
@@ -299,66 +399,98 @@ local do_setmatrix do
     c, d = projected(m, c, d, 0)
     e, f = projected(m, e, f, 1)
     m[1], m[2], m[3], m[4], m[5], m[6] = a, b, c, d, e, f
-  end
+  end)
 end
-local function do_save(prop, p, n, x, y, outer)
+local save_whatsit = declare_whatsit('pdf_save', function(prop, p, n, x, y, outer)
   pdf.write('page', 'q', x, y, p)
   local lastmatrix = p.matrix
   p.matrix = {[0] = lastmatrix, table.unpack(lastmatrix)}
-end
-local function do_restore(prop, p, n, x, y, outer)
+end)
+local restore_whatsit = declare_whatsit('pdf_restore', function(prop, p, n, x, y, outer)
   -- TODO: Check x, y
   pdf.write('page', 'Q', x, y, p)
   p.matrix = p.matrix[0]
-end
-local function do_dest(prop, p, n, x, y)
-  -- TODO: Apply matrix
+end)
+local dest_whatsit = declare_whatsit('pdf_dest', function(prop, p, n, x, y)
+  if not prop then
+    tex.error('Invalid pdf_dest whatsit', {"A pdf_dest whatsit did not contain all necessary \z
+        parameters. Maybe your code hasn't been adapted to LuaMetaLaTeX yet?"})
+  end
   assert(cur_page, "Destinations can not appear outside of a page")
   local id = prop.dest_id
   local dest_type = prop.dest_type
+  local off = pdfvariable.linkmargin
   local data
   if dest_type == "xyz" then
+    local x, y = projected(p.matrix, x, y)
     local zoom = prop.xyz_zoom
     if zoom then
-      data = string.format("[%i 0 R/XYZ %.5f %.5f %.3f]", cur_page, sp2bp(x), sp2bp(y), prop.zoom/1000)
+      data = string.format("[%i 0 R/XYZ %.5f %.5f %.3f]", cur_page, to_bp(x-off), to_bp(y+off), prop.zoom/1000)
     else
-      data = string.format("[%i 0 R/XYZ %.5f %.5f null]", cur_page, sp2bp(x), sp2bp(y))
+      data = string.format("[%i 0 R/XYZ %.5f %.5f null]", cur_page, to_bp(x-off), to_bp(y+off))
     end
   elseif dest_type == "fitr" then
-    data = string.format("[%i 0 R/FitR %.5f %.5f %.5f %.5f]", cur_page, sp2bp(x), sp2bp(y + prop.depth), sp2bp(x + prop.width), sp2bp(y - prop.height))
+    local m = p.matrix
+    local llx, lly = projected(x, x - prop.depth)
+    local lrx, lry = projected(x+prop.width, x - prop.depth)
+    local ulx, uly = projected(x, x + prop.height)
+    local urx, ury = projected(x+prop.width, x + prop.height)
+    local left, lower, right, upper = math.min(llx, lrx, ulx, urx), math.min(lly, lry, uly, ury),
+                                      math.max(llx, lrx, ulx, urx), math.max(lly, lry, uly, ury)
+    data = string.format("[%i 0 R/FitR %.5f %.5f %.5f %.5f]", cur_page, to_bp(left-off), to_bp(lower-off), to_bp(right+off), to_bp(upper+off))
   elseif dest_type == "fit" then
     data = string.format("[%i 0 R/Fit]", cur_page)
   elseif dest_type == "fith" then
-    data = string.format("[%i 0 R/FitH %.5f]", cur_page, sp2bp(y))
+    local x, y = projected(p.matrix, x, y)
+    data = string.format("[%i 0 R/FitH %.5f]", cur_page, to_bp(y+off))
   elseif dest_type == "fitv" then
-    data = string.format("[%i 0 R/FitV %.5f]", cur_page, sp2bp(x))
+    local x, y = projected(p.matrix, x, y)
+    data = string.format("[%i 0 R/FitV %.5f]", cur_page, to_bp(x-off))
   elseif dest_type == "fitb" then
     data = string.format("[%i 0 R/FitB]", cur_page)
   elseif dest_type == "fitbh" then
-    data = string.format("[%i 0 R/FitBH %.5f]", cur_page, sp2bp(y))
+    local x, y = projected(p.matrix, x, y)
+    data = string.format("[%i 0 R/FitBH %.5f]", cur_page, to_bp(y+off))
   elseif dest_type == "fitbv" then
-    data = string.format("[%i 0 R/FitBV %.5f]", cur_page, sp2bp(x))
+    local x, y = projected(p.matrix, x, y)
+    data = string.format("[%i 0 R/FitBV %.5f]", cur_page, to_bp(x-off))
   end
   if pfile:written(dests[id]) then
     texio.write_nl(string.format("Duplicate destination %q", id))
   else
-    dests[id] = pfile:indirect(dests[id], data)
+    dests[id] = pfile:indirect(dests[id], strip_floats(data))
+  end
+end)
+local refobj_whatsit = declare_whatsit('pdf_refobj', function(prop, p, n, x, y)
+  if not prop then
+    tex.error('Invalid pdf_refobj whatsit', {"A pdf_refobj whatsit did not reference any object. \z
+        Maybe your code hasn't been adapted to LuaMetaLaTeX yet?"})
+    return
   end
-end
-local function do_refobj(prop, p, n, x, y)
   pfile:reference(prop.obj)
-end
-local function do_literal(prop, p, n, x, y)
+end)
+local literal_whatsit = declare_whatsit('pdf_literal', function(prop, p, n, x, y)
+  if not prop then
+    tex.error('Invalid pdf_literal whatsit', "A pdf_literal whatsit did not contain a literal to be inserted. \z
+        Maybe your code hasn't been adapted to LuaMetaLaTeX yet?")
+    return
+  end
   pdf.write(prop.mode, prop.data, x, y, p)
-end
-local function do_colorstack(prop, p, n, x, y)
+end)
+local colorstack_whatsit = declare_whatsit('pdf_colorstack', function(prop, p, n, x, y)
+  if not prop then
+    tex.error('Invalid pdf_colorstack whatsit', {"A pdf_colorstack whatsit did not contain all necessary \z
+        parameters. Maybe your code hasn't been adapted to LuaMetaLaTeX yet?"})
+    return
+  end
   local colorstack = prop.colorstack
   local stack
   if p.is_page then
     stack = colorstack.page_stack
-  elseif prop.last_form == resources then
+  elseif colorstack.last_form == p.resources then
     stack = colorstack.form_stack
   else
+    colorstack.last_form = p.resources
     stack = {prop.default}
     colorstack.form_stack = stack
   end
@@ -371,28 +503,33 @@ local function do_colorstack(prop, p, n, x, y)
     stack[#stack] = prop.data
   end
   pdf.write(colorstack.mode, stack[#stack], x, y, p)
-end
+end)
 local function write_colorstack()
   local idx = token.scan_int()
   local colorstack = colorstacks[idx + 1]
   if not colorstack then
-    error[[Undefined colorstack]]
+    tex.error('Undefined colorstack', {"The requested colorstack is not initialized. \z
+        This probably means that you forgot to run \\pdffeedback colorstackinit or \z
+        that you specified the wrong index. I will continue with colorstack 0."})
+    colorstack = colorstacks[1]
   end
   local action = token.scan_keyword'pop' and 'pop'
               or token.scan_keyword'set' and 'set'
               or token.scan_keyword'current' and 'current'
               or token.scan_keyword'push' and 'push'
   if not action then
-    error[[Missing action specifier for colorstack command]]
+    tex.error('Missing action specifier for colorstack', {
+        "I don't know what you want to do with this colorstack. I would have expected pop/set/current or push here. \z
+        I will ignore this command."})
+    return
   end
   local text
   if action == "push" or "set" then
     text = token.scan_string()
     -- text = token.to_string(token.scan_tokenlist()) -- Attention! This should never be executed in an expand-only context
   end
-  local whatsit = node.new(whatsit_id, whatsits.pdf_colorstack)
+  local whatsit = node.new(whatsit_id, colorstack_whatsit)
   node.setproperty(whatsit, {
-      handle = do_colorstack,
       colorstack = colorstack,
       action = action,
       data = text,
@@ -416,9 +553,15 @@ local function scan_action()
     file = token.scan_keyword'file' and token.scan_string(),
   }
   if token.scan_keyword'page' then
-    error[[TODO]]
+    assert(action_type == 1)
+    local page = token.scan_int()
+    if page <= 0 then
+      error[[page must be positive in action specified]]
+    end
+    action.page = page
+    action.tokens = token.scan_string()
   elseif token.scan_keyword'num' then
-    if action.file and action_type == 3 then
+    if action.file and action_type == 1 then
       error[[num style GoTo actions must be internal]]
     end
     action.id = token.scan_int()
@@ -476,57 +619,54 @@ token.luacmd("pdfextension", function(_, imm)
   elseif token.scan_keyword"literal" then
     local mode = scan_literal_mode()
     local literal = token.scan_string()
-    local whatsit = node.new(whatsit_id, whatsits.pdf_literal)
+    local whatsit = node.new(whatsit_id, literal_whatsit)
     node.setproperty(whatsit, {
-        handle = do_literal,
         mode = mode,
         data = literal,
       })
     node.write(whatsit)
   elseif token.scan_keyword"startlink" then
     local pfile = get_pfile()
-    local whatsit = node.new(whatsit_id, whatsits.pdf_start_link)
+    local whatsit = node.new(whatsit_id, start_link_whatsit)
     local attr = token.scan_keyword'attr' and token.scan_string() or ''
     local action = scan_action()
     local objnum = pfile:getobj()
     lastannot = num
     node.setproperty(whatsit, {
-        handle = do_start_link,
         link_attr = attr,
         action = action,
         objnum = objnum,
       })
     node.write(whatsit)
   elseif token.scan_keyword"endlink" then
-    local whatsit = node.new(whatsit_id, whatsits.pdf_end_link)
-    node.setproperty(whatsit, {
-        handle = do_end_link,
-      })
+    local whatsit = node.new(whatsit_id, end_link_whatsit)
     node.write(whatsit)
   elseif token.scan_keyword"save" then
-    local whatsit = node.new(whatsit_id, whatsits.pdf_save)
-    node.setproperty(whatsit, {
-        handle = do_save,
-      })
+    local whatsit = node.new(whatsit_id, save_whatsit)
     node.write(whatsit)
   elseif token.scan_keyword"setmatrix" then
     local matrix = token.scan_string()
-    local whatsit = node.new(whatsit_id, whatsits.pdf_setmatrix)
+    local whatsit = node.new(whatsit_id, setmatrix_whatsit)
     node.setproperty(whatsit, {
-        handle = do_setmatrix,
         data = matrix,
       })
     node.write(whatsit)
   elseif token.scan_keyword"restore" then
-    local whatsit = node.new(whatsit_id, whatsits.pdf_restore)
-    node.setproperty(whatsit, {
-        handle = do_restore,
-      })
+    local whatsit = node.new(whatsit_id, restore_whatsit)
     node.write(whatsit)
   elseif token.scan_keyword"info" then
     infodir = infodir .. token.scan_string()
   elseif token.scan_keyword"catalog" then
     catalogdir = catalogdir .. ' ' .. token.scan_string()
+    if token.scan_keyword'openaction' then
+      if catalog_openaction then
+        tex.error("Duplicate openaction", {"Only one use of \\pdfextension catalog is allowed to \z
+            have an openaction."})
+      else
+        local action = scan_action()
+        catalog_openaction = get_action_attr(get_pfile(), action)
+      end
+    end
   elseif token.scan_keyword"names" then
     namesdir = namesdir .. ' ' .. token.scan_string()
   elseif token.scan_keyword"obj" then
@@ -555,10 +695,9 @@ token.luacmd("pdfextension", function(_, imm)
     end
   elseif token.scan_keyword"refobj" then
     local num = token.scan_int()
-    local whatsit = node.new(whatsit_id, whatsits.pdf_refobj)
+    local whatsit = node.new(whatsit_id, refobj_whatsit)
     node.setproperty(whatsit, {
         obj = num,
-        handle = do_refobj,
       })
     node.write(whatsit)
   elseif token.scan_keyword"outline" then
@@ -594,10 +733,9 @@ token.luacmd("pdfextension", function(_, imm)
     else
       error[[Unsupported id type]]
     end
-    local whatsit = node.new(whatsit_id, whatsits.pdf_dest)
+    local whatsit = node.new(whatsit_id, dest_whatsit)
     local prop = {
       dest_id = id,
-      handle = do_dest,
     }
     node.setproperty(whatsit, prop)
     if token.scan_keyword'xyz' then
@@ -640,9 +778,129 @@ token.luacmd("pdfextension", function(_, imm)
       error[[Unsupported dest type]]
     end
     node.write(whatsit)
+  elseif token.scan_keyword'mapline' then
+    fontmap.mapline(token.scan_string())
   else
   -- The following error message gobbles the next word as a side effect.
   -- This is intentional to make error-recovery easier.
     error(string.format("Unknown PDF extension %s", token.scan_word()))
   end
 end, "protected")
+local imglib = require'luametalatex-pdf-image'
+local imglib_node = imglib.node
+local imglib_write = imglib.write
+local imglib_immediatewrite = imglib.immediatewrite
+img = {
+  new = imglib.new,
+  scan = imglib.scan,
+  node = function(img, pfile) return imglib_node(pfile or get_pfile(), img) end,
+  write = function(img, pfile) return imglib_write(pfile or get_pfile(), img) end,
+  immediatewrite = function(img, pfile) return imglib_immediatewrite(pfile or get_pfile(), img) end,
+}
+
+local lastimage = -1
+local lastimagepages = -1
+
+-- These are very minimal right now but LaTeX isn't using the scaling etc. stuff anyway.
+token.luacmd("saveimageresource", function(imm)
+  local attr = token.scan_keyword'attr' and token.scan_string() or nil
+  local page = token.scan_keyword'page' and token.scan_int() or nil
+  local userpassword = token.scan_keyword'userpassword' and token.scan_string() or nil
+  local ownerpassword = token.scan_keyword'ownerpassword' and token.scan_string() or nil
+  -- local colorspace = token.scan_keyword'colorspace' and token.scan_int() or nil -- Doesn't make sense for PDF
+  local pagebox = token.scan_keyword'mediabox' and 'media'
+               or token.scan_keyword'cropbox' and 'crop'
+               or token.scan_keyword'bleedbox' and 'bleed'
+               or token.scan_keyword'trimbox' and 'trim'
+               or token.scan_keyword'artbox' and 'art'
+               or nil
+  local filename = token.scan_string()
+  local img = imglib.scan{
+    attr = attr,
+    page = page,
+    userpassword = userpassword,
+    ownerpassword = ownerpassword,
+    pagebox = pagebox,
+    filename = filename,
+  }
+  local pfile = get_pfile()
+  lastimage = imglib.get_num(pfile, img)
+  lastimagepages = img.pages or 1
+  if imm == 'immediate' then
+    imglib_immediatewrite(pfile, img)
+  end
+end, "protected")
+
+token.luacmd("useimageresource", function()
+  local pfile = get_pfile()
+  local img = assert(imglib.from_num(token.scan_int()))
+  imglib_write(pfile, img)
+end, "protected")
+
+local value_values = token.values'value'
+for i=0,#value_values do
+  value_values[value_values[i]] = i
+end
+local integer_code = value_values.integer
+
+token.luacmd("lastsavedimageresourceindex", function()
+  return integer_code, lastimage
+end, "protected", "value")
+
+token.luacmd("lastsavedimageresourcepages", function()
+  return integer_code, lastimagepages
+end, "protected", "value")
+
+local savedbox = require'luametalatex-pdf-savedbox'
+local savedbox_save = savedbox.save
+function tex.saveboxresource(n, attr, resources, immediate, type, margin, pfile)
+  if not node.type(n) then
+    n = tonumber(n)
+    if not n then
+      error[[Invalid argument to saveboxresource]]
+    end
+    token.put_next(token.create'box', token.new(n, token.command_id'char_given'))
+    n = token.scan_list()
+  end
+  margin = margin or tex.sp'1bp' -- FIXME: default margin variable
+  return savedbox_save(pfile or get_pfile(), n, attr, resources, immediate, type, margin, fontdirs, usedglyphs)
+end
+tex.useboxresource = savedbox.use
+
+local lastbox = -1
+
+token.luacmd("saveboxresource", function(imm)
+  local type
+  if token.scan_keyword'type' then
+    texio.write_nl('XForm type attribute ignored')
+    type = token.scan_int()
+  end
+  local attr = token.scan_keyword'attr' and token.scan_string() or nil
+  local resources = token.scan_keyword'resources' and token.scan_string() or nil
+  local margin = token.scan_keyword'margin' and token.scan_dimen() or nil
+  local box = token.scan_int()
+
+  local index = tex.saveboxresource(box, attr, resources, imm == 'immediate', type, margin)
+  lastbox = index
+end, "protected")
+
+token.luacmd("useboxresource", function()
+  local width, height, depth
+  while true do
+    if token.scan_keyword'width' then
+      width = token.scan_dimen()
+    elseif token.scan_keyword'height' then
+      height = token.scan_dimen()
+    elseif token.scan_keyword'depth' then
+      depth = token.scan_dimen()
+    else
+      break
+    end
+  end
+  local index = token.scan_int()
+  node.write((tex.useboxresource(index, width, height, depth)))
+end, "protected")
+
+token.luacmd("lastsavedboxresourceindex", function()
+  return integer_code, lastbox
+end, "protected", "value")

luametalatex-baseregisters.lua

@@ -0,0 +1,193 @@
+local value_values = token.values'value'
+for i=0,#value_values do
+  value_values[value_values[i]] = i
+end
+local count_code = value_values.integer
+local dimen_code = value_values.dimension
+
+local set_local = require'luametalatex-local'
+
+local texmeta = getmetatable(tex)
+local texmetaoldindex = texmeta.__index
+local texmetaoldnewindex = texmeta.__newindex
+
+local tex_variables = __luametalatex__preserved_tex_variables or {}
+__luametalatex__preserved_tex_variables = nil
+
+function texmeta.__index(t, k)
+  return tex_variables[k] or texmetaoldindex(t, k)
+end
+function texmeta.__newindex(t, k, v)
+  if tex_variables[k] then
+    return set_local(tex_variables, k, v)
+  else
+    return texmetaoldnewindex(t, k, v)
+  end
+end
+
+local function tex_variable(value, scanner, name, default)
+  token.luacmd(name, function(_, scanning)
+    if scanning == 'value' then
+      return value, tex_variables[name]
+    else
+      token.scan_keyword'='
+      return set_local(tex_variables, name, scanner(), scanning == 'global')
+    end
+  end, 'global', 'protected', 'value')
+  if status.ini_version then
+    tex_variables[name] = default
+  end
+end
+
+local real_pdf_variables = __luametalatex__preserved_real_pdf_variables or {}
+__luametalatex__preserved_real_pdf_variables = nil
+local pdf_variable_names = {}
+local pdf_toks_map = {}
+local pdf_variables = setmetatable(pdf.variable, {
+  __index = function(_, k)
+    local v = real_pdf_variables[k]
+    if v then return v end
+    v = pdf_toks_map[k]
+    if v then
+      return tex.toks[v]
+    end
+  end,
+  __newindex = function(_, k, v)
+    if real_pdf_variables[k] then
+      return set_local(real_pdf_variables, k, v)
+    end
+    local toks = pdf_toks_map[k]
+    if toks then
+      tex.toks[toks] = v
+    end
+  end,
+})
+pdf.variable_names = pdf_variable_names
+
+local pdf_toks
+if status.ini_version then
+  local pdf_toks_list = {}
+  function pdf_toks(name, default)
+    pdf_variable_names[#pdf_variable_names+1] = name
+    pdf_toks_list[#pdf_toks_list+1] = {name, default}
+  end
+  function initialize_pdf_toks()
+    for i=1,#pdf_toks_list do
+      local entry = pdf_toks_list[i]
+      local csname = 'pdfvariable  ' .. entry[1]
+      token.set_char(csname, 0) -- Ensure that csname exists
+      local t = token.create(csname)
+      tex.runtoks(function()
+        token.put_next(token.create'newtoks', t)
+      end)
+      pdf_toks_map[entry[1]] = t.index
+      tex.toks[t.index] = entry[2]
+    end
+  end
+else
+  function pdf_toks(name, default)
+    pdf_variable_names[#pdf_variable_names+1] = name
+    local t = token.create('pdfvariable  ' .. name)
+    pdf_toks_map[name] = t.index
+    tex.toks[t.index] = default
+  end
+end
+
+local function pdf_variable(value, scanner, name, default, force_default)
+  pdf_variable_names[#pdf_variable_names+1] = name
+  token.luacmd('pdfvariable  ' .. name, function(_, scanning)
+    if scanning == 'value' then
+      return value, real_pdf_variables[name]
+    elseif force_default then
+      token.scan_keyword'='
+      local new = scanner()
+      if new ~= default then
+        texio.write_nl('term and log', string.format("Unsupported PDF variable: \z
+            %q is not supported and fixed to %i, but you tried to set it to %i", name, default, new))
+      end
+    else
+      token.scan_keyword'='
+      return set_local(real_pdf_variables, name, scanner(), scanning == 'global')
+    end
+  end, 'global', 'protected', 'value')
+  if status.ini_version then
+    real_pdf_variables[name] = default
+  end
+end
+
+tex_variable(count_code, token.scan_int, 'suppressfontnotfounderror', 0)
+tex_variable(count_code, token.scan_int, 'outputmode', 1) -- The "traditional" default would be 0,
+                                                            -- but we do not actually support that.
+tex_variable(dimen_code, token.scan_dimen, 'pageheight', tex.sp'297mm')
+tex_variable(dimen_code, token.scan_dimen, 'pagewidth', tex.sp'210mm')
+
+tex_variable(count_code, token.scan_int, 'bodydirection', 0)
+tex_variable(count_code, token.scan_int, 'pagedirection', 0)
+
+pdf_variable(dimen_code, token.scan_dimen, 'horigin', tex.sp'1in')
+pdf_variable(dimen_code, token.scan_dimen, 'vorigin', tex.sp'1in')
+pdf_variable(dimen_code, token.scan_dimen, 'linkmargin', tex.sp'0pt')
+pdf_variable(dimen_code, token.scan_dimen, 'destmargin', tex.sp'0pt')
+pdf_variable(dimen_code, token.scan_dimen, 'threadmargin', tex.sp'0pt') -- We don't support threads, so this isn't doing anything
+pdf_variable(count_code, token.scan_int, 'majorversion', 1)
+pdf_variable(count_code, token.scan_int, 'minorversion', 7)
+pdf_variable(count_code, token.scan_int, 'compresslevel', 9)
+pdf_variable(count_code, token.scan_int, 'objcompresslevel', 3)
+
+pdf_variable(count_code, token.scan_int, 'decimaldigits', 4, true) -- Will probably stay fixed, but should be more consistent
+pdf_variable(count_code, token.scan_int, 'gentounicode', 0, true) -- We expect the fontloader to generade tounicode tables. Might change at some point
+-- These two are ignored, but that is consistent with pdfTeX as long as imageapplygamma is 0:
+pdf_variable(count_code, token.scan_int, 'gamma', 1000)
+pdf_variable(count_code, token.scan_int, 'imagegamma', 1000)
+pdf_variable(count_code, token.scan_int, 'imageapplygamma', 0, true)
+pdf_variable(count_code, token.scan_int, 'imagehicolor', 1, true) -- We don't consider ancient PDF versions, no no reason to strip images
+pdf_variable(count_code, token.scan_int, 'imageaddfilename', 0, true) -- Could be added, but I never saw a reason for this anyway.
+pdf_variable(count_code, token.scan_int, 'inclusionerrorlevel', -1, true) -- FIXME: At least a warning should be supported
+pdf_variable(count_code, token.scan_int, 'inclusioncopyfonts', 0, true) -- Would be fragile and restrict our ability to use "creative" font constructs
+pdf_variable(count_code, token.scan_int, 'uniqueresname', 0, true) -- I add this if you show me a usecase
+pdf_variable(count_code, token.scan_int, 'pagebox', 2, true) -- TODO (1: media, 2: crop, 3: bleed, 4: trim, 5: art
+pdf_variable(count_code, token.scan_int, 'forcepagebox', 0, true) -- Considered obsolete even in pdfTeX
+pdf_variable(count_code, token.scan_int, 'imageresolution', 72, true) -- TODO Also 0 should be the same as 72 ?!?!?!?
+
+pdf_variable(count_code, token.scan_int, 'pkresolution', 1200) -- Original default is 72, but that's crazy
+pdf_variable(count_code, token.scan_int, 'pkfixeddpi', 0) -- TODO: Implemented, but even when set to one, font sharing doesn't adapt yet.
+                                                          -- Changing that is complicated because it has to be known pretty early.
+pdf_toks('pkmode', '')
+
+pdf_toks('pageattr', '')
+pdf_toks('pagesattr', '')
+pdf_toks('pageresources', '')
+
+-- The following two are special: The should have force_default=true because we ignore them,
+-- but they are token lists so that doesn't really work. Instead, we check during shipout that
+-- the variables are empty. TODO: Find a nicer solution
+-- The reason for not implementing them is that XForm specific resources and attributes make
+-- much more sense, so these two generic ones are useless and error-prone.
+pdf_toks('xformresources', '')
+pdf_toks('xformattr', '')
+
+function tex.getbodydir() return tex.bodydirection end
+function tex.getpagedir() return tex.pagedirection end
+function tex.setbodydir(i) tex.bodydirection = i end
+function tex.setpagedir(i) tex.pagedirection = i end
+local dir_regs = require 'luametalatex-dir-registers'
+dir_regs 'textdir'
+dir_regs 'bodydir'
+dir_regs 'pagedir'
+
+if status.ini_version then
+  -- Run in pre_dump callback:
+  lua.prepared_code[#lua.prepared_code+1] = function()
+    local settings = " "
+    for k,v in next, {__luametalatex__preserved_tex_variables = tex_variables,
+                      __luametalatex__preserved_real_pdf_variables = real_pdf_variables,} do
+      local entries = {}
+      for kk,vv in next, v do
+        -- entries[#entries+1] = string.format("[%q=%i],", kk, vv) -- If we ever get more compicated names here
+        entries[#entries+1] = string.format("%s=%i,", kk, vv)
+      end
+      settings = string.format("%s%s={%s}", settings, k, table.concat(entries))
+    end
+    return settings
+  end
+end

luametalatex-baseregisters.tex

@@ -1,124 +0,0 @@
-\begingroup
-\catcode`\^^^^ffff=11
-\catcode`\@=11
-\toks0{%
-  do
-  local function frozen(s)
-    local t = token.create(s)
-    return token.new(t.mode, t.command)
-  end
-  local dimen_cmd = token.command_id'assign_dimen'
-  local count_cmd = token.command_id'assign_int'
-  local toks_cmd = token.command_id'assign_toks'
-  local tex_params = {}
-  local texmeta = getmetatable(tex)
-  local texmetaoldindex = texmeta.__index
-  local texmetaoldnewindex = texmeta.__newindex
-  function texmeta.__index(t, k)
-    local v = tex_params[k]
-    if v then
-      if v.command == count_cmd then
-        return tex.count[v.index]
-      elseif v.command == dimen_cmd then
-        return tex.dimen[v.index]
-      elseif v.command == toks_cmd then
-        return tex.toks[v.index]
-      end
-    else
-      return texmetaoldindex(t, k)
-    end
-  end
-  function texmeta.__newindex(t, k, v)
-    local p = tex_params[k]
-    if p then
-      if p.command == count_cmd then
-        tex.count[p.index] = v
-      elseif p.command == dimen_cmd then
-        tex.dimen[p.index] = v
-      elseif p.command == toks_cmd then
-        tex.toks[p.index] = v
-      end
-    else
-      return texmetaoldnewindex(t, k, v)
-    end
-  end
-  local pdf_params = {}
-  pdf.variable_tokens = pdf_params
-  pdf.variable = setmetatable({}, {
-    __index = function(t, k)
-      local v = pdf_params[k]
-      if v then
-        if v.command == count_cmd then
-          return tex.count[v.index]
-        elseif v.command == dimen_cmd then
-          return tex.dimen[v.index]
-        elseif v.command == toks_cmd then
-          return tex.toks[v.index]
-        end
-      else
-        return texmetaoldindex(t, k)
-      end
-    end,
-    __newindex = function(t, k, v)
-      local p = pdf_params[k]
-      if p then
-        if p.command == count_cmd then
-          tex.count[p.index] = v
-        elseif p.command == dimen_cmd then
-          tex.dimen[p.index] = v
-        elseif p.command == toks_cmd then
-          tex.toks[p.index] = v
-        end
-      else
-        return texmetaoldnewindex(t, k, v)
-      end
-    end,
-  })
-}
-\def\InternalAlloc#1#2#3#4#5{%
-  \csname new#3\endcsname#1%
-  \global#1=#5\relax
-  \etoksapp0{#2_params["\luaescapestring{#4}"] = frozen"\luaescapestring{\csstring#1}"}
-}
-\def\internalAlloc#1#2#3{%
-  \expandafter\InternalAlloc\csname ^^^^fffe#3@#1\endcsname{#1}{#2}{#3}%
-}
-\def\texAlloc#1#2{%
-  \expandafter\InternalAlloc\csname #2\endcsname{tex}{#1}{#2}%
-}
-\def\pdfAlloc{%
-  \internalAlloc{pdf}%
-}
-\texAlloc{count}{suppressfontnotfounderror}{0}
-\texAlloc{count}{outputmode}{1} % The "traditional" default would be 0,
-                                % but we do not actually support that.
-\texAlloc{dimen}{pageheight}{297mm}
-\texAlloc{dimen}{pagewidth}{210mm}
-\pdfAlloc{dimen}{horigin}{1in}
-\pdfAlloc{dimen}{vorigin}{1in}
-\pdfAlloc{dimen}{linkmargin}{0pt}
-\pdfAlloc{count}{majorversion}{1}
-\pdfAlloc{count}{minorversion}{7}
-\pdfAlloc{count}{compresslevel}{0} % 0 is actually the only supported value right now, so this is basically ignored
-\pdfAlloc{count}{objcompresslevel}{0} % see above
-\pdfAlloc{toks}{pageresources}{{}}
-
-\texAlloc{count}{bodydirection}{0}
-\texAlloc{count}{pagedirection}{0}
-\etoksapp0{
-  function tex.getbodydir() return tex.bodydirection end
-  function tex.getpagedir() return tex.pagedirection end
-  function tex.setbodydir(i) tex.bodydirection = i end
-  function tex.setpagedir(i) tex.pagedirection = i end
-  local dir_regs = require 'luametalatex-dir-registers'
-  dir_regs 'textdir'
-  dir_regs 'bodydir'
-  dir_regs 'pagedir'
-  end
-}
-
-\directlua{
-  lua.prepared_code[\csstring#lua.prepared_code+1] = tex.toks[0]
-  \the\toks0
-}
-\endgroup

luametalatex-bit32.lua

@@ -16,6 +16,6 @@ return {
     return (i ^ j) & mask32
   end,
   extract = function(v, shift, count)
-    return ((bit32 & v) >> shift) & ((1<<count)-1)
+    return ((mask32 & v) >> shift) & ((1<<count)-1)
   end,
 }

luametalatex-build-bytecode.lua

@@ -0,0 +1,28 @@
+local concat = table.concat
+local format = string.format
+local ioopen = io.open
+local assert = assert
+local ipairs = ipairs
+
+local _ENV = {}
+
+local first, later =
+  'local __hidden_local__package_preload__=package.preload',
+  '\n__hidden_local__package_preload__[%q]=function(...)%s\nend'
+
+first = first .. later
+
+local list = {}
+
+return function(t)
+  local length = #t
+  local tmpl = first
+  for i, mod in ipairs(t) do
+    local name, f = mod[1], assert(ioopen(mod[2], 'r'))
+    local data = f:read'a'
+    f:close()
+    list[i] = format(tmpl, name, data)
+    tmpl = later
+  end
+  return concat(list, nil, 1, length)
+end

luametalatex-callbacks.lua

@@ -0,0 +1,84 @@
+-- Two callbacks are defined in other files: pre_dump in lateinit and find_fmt_file in init
+
+local read_tfm = font.read_tfm
+local font_define = font.define
+local callback_register = callback.register
+
+if status.ini_version then
+  callback_register('define_font', function(name, size)
+    local f = read_tfm(name, size)
+    if not f then return end
+    local id = font_define(f)
+    lua.prepared_code[#lua.prepared_code+1] = string.format("assert(%i == font.define(font.read_tfm(%q, %i)))", id, name, size)
+    return id
+  end)
+else
+  callback_register('define_font', function(name, size)
+    local f = read_tfm(name, size)
+    if not f then return end
+    return font.define(f)
+  end)
+end
+callback_register('find_log_file', function(name) return name end)
+do
+  local function normal_find_data_file(name)
+    return kpse.find_file(name, 'tex', true)
+  end
+  if status.ini_version then
+    function unhook_expl()
+      callback_register('find_data_file', normal_find_data_file)
+    end
+    callback_register('find_data_file', function(name)
+      if name == 'ltexpl.ltx' then
+        name = 'luametalatex-ltexpl-hook'
+      end
+      return normal_find_data_file(name)
+    end)
+  else
+    callback_register('find_data_file', normal_find_data_file)
+  end
+end
+-- callback_register('read_data_file', function(name) error[[TODO]]return kpse.find_file(name, 'tex', true) end)
+callback_register('open_data_file', function(name)
+  local f = io.open(name)
+  return setmetatable({
+    reader = function()
+      local line = f:read()
+      return line
+    end,
+    close = function()error[[1]] return f:close() end,
+  }, {
+    __gc = function()f:close()end,
+  })
+end)
+callback_register('handle_error_hook', function()
+  repeat
+    texio.write_nl'? '
+    local line = io.read()
+    if not line then
+      tex.fatalerror'End of line encountered on terminal'
+    end
+    if line == "" then return 3 end
+    local first = line:sub(1,1):upper()
+    if first == 'H' then
+      texio.write(tex.gethelptext() or "Sorry, I don't know how to help in this situation.\n\z
+        Maybe you should try asking a human?")
+    elseif first == 'I' then
+      line = line:sub(2)
+      tex.runtoks(function()
+        tex.sprint(token.scan_token(), line)
+      end)
+      return 3
+    elseif first == 'Q' then texio.write'OK, entering \\batchmode...\n' return 0
+    elseif first == 'R' then texio.write'OK, entering \\nonstopmode...\n' return 1
+    elseif first == 'S' then texio.write'OK, entering \\scrollmode...\n' return 2
+    elseif first == 'X' then return -1
+    else
+      texio.write'Type <return> to proceed, S to scroll future error messages,\
+\z      R to run without stopping, Q to run quietly,\
+\z      I to insert something,\
+\z      H for help, X to quit.'
+    end
+  until false
+  return 3
+end)

luametalatex-dir-registers.lua

@@ -11,16 +11,13 @@ function tex.gettextdir() return tex.textdirection end
 function tex.getlinedir() return tex.linedirection end
 function tex.getmathdir() return tex.mathdirection end
 function tex.getpardir()  return tex.pardirection  end
--- local integer_code = value_values.none
 local integer_code = value_values.integer
-local functions = lua.get_functions_table()
-local lua_call_cmd = token.command_id'lua_call'
 local function set_xdir(id, scanning)
-  -- local name = names[id]
-  if scanning then
+  if scanning == 'value' then
+    print(scanning)
     return integer_code, getters[id]()
-    -- return integer_code, tex[name .. 'ection']
   end
+  -- local global = scanning == 'global'
   local value
   if token.scan_keyword'tlt' then
     value = 0
@@ -30,14 +27,12 @@ local function set_xdir(id, scanning)
     value = token.scan_int()
   end
   setters[id](value)
-  -- tex["set" .. name](value)
 end
 return function(name)
   local getter = tex["get" .. name]
   local setter = tex["set" .. name]
   assert(getter and setter, "direction parameter undefined")
   local idx = token.luacmd(name, set_xdir, "protected", "global", "value")
-  -- names[idx] = name
   getters[idx] = getter
   setters[idx] = setter
   return idx

luametalatex-firstcode.lua

@@ -1,55 +1,4 @@
-local functions = lua.getfunctionstable()
--- I am not sure why this is necessary, but otherwise LuaMetaTeX resets
--- the functions table every time the getter is called
-function lua.get_functions_table() return functions end
-local set_lua = token.set_lua
--- There are two approaches to manage luafunctions ids without triggering
--- issues with ltluatex assigned numbers: Originally we assigned numbers
--- starting with 1, then switched to luatexbase ASAP and synchronised both
--- numbering schemes. But there is a useful quirk in ltluatex's luafunction
--- allocator: It only uses numbers upto 65535, so we can just use bigger
--- numbers. (This might have negative repercussins on performance because it
--- probably doesn't store the function in the array part of the Lua table.
--- Let's reconsider if this ever becomes a problem.
--- local new_luafunction = luatexbase.new_luafunction
-local predefined_luafunctions = status.ini_version and 65535 -- 1<<16 - 1 -- We start with 1<<16
-local function new_luafunction(name)
-  if predefined_luafunctions then
-    predefined_luafunctions = predefined_luafunctions + 1
-    return predefined_luafunctions
-  else
-    error[[Here only preexisting luafunctions should be set]]
-  end
-end
-local undefined_cmd = token.command_id'undefined_cs'
 local lua_call_cmd = token.command_id'lua_call'
-local lua_value_cmd = token.command_id'lua_value'
-local lua_expandable_call_cmd = token.command_id'lua_expandable_call'
-function token.luacmd(name, func, ...)
-  local idx
-  local tok = token.create(name)
-  local cmd = tok.command
-  if cmd == lua_value_cmd then
-    idx = tok.mode
-  elseif cmd == lua_call_cmd then
-    idx = tok.mode
-  elseif cmd == lua_expandable_call_cmd then
-    idx = tok.mode
-  elseif ... == 'force' then
-    idx = new_luafunction(name)
-    set_lua(name, idx, select(2, ...))
-  elseif cmd == undefined_cmd then
-    idx = new_luafunction(name)
-    set_lua(name, idx, ...)
-  else
-    error(tok.cmdname)
-  end
-  if functions[idx] then
-    error[[Already defined]]
-  end
-  functions[idx] = func
-  return idx
-end
 local properties = node.direct.get_properties_table()
 node.direct.properties = properties
 function node.direct.get_properties_table()
@@ -63,50 +12,8 @@ end
 --     end
 --   })
 
+local new_whatsit = require'luametalatex-whatsits'.new
 local whatsit_id = node.id'whatsit'
-local whatsits = {
-  [0] = "open",
-        "write",
-        "close",
-        "special",
-        nil,
-        nil,
-        "save_pos",
-        "late_lua",
-        "user_defined",
-        nil,
-        nil,
-        nil,
-        nil,
-        nil,
-        nil,
-        nil,
-        "pdf_literal",
-        "pdf_refobj",
-        "pdf_annot",
-        "pdf_start_link",
-        "pdf_end_link",
-        "pdf_dest",
-        "pdf_action",
-        "pdf_thread",
-        "pdf_start_thread",
-        "pdf_end_thread",
-        "pdf_thread_data",
-        "pdf_link_data",
-        "pdf_colorstack",
-        "pdf_setmatrix",
-        "pdf_save",
-        "pdf_restore",
-}
-whatsits[whatsits[0]] = 0
-for i = 0,#whatsits do
-  local v = whatsits[i]
-  if v then
-    whatsits[v] = i
-  end
-end
-function node.whatsits() return whatsits end
-function node.subtype(s) return type(s) == "string" and whatsits[s] or nil end
 local spacer_cmd, relax_cmd = token.command_id'spacer', token.command_id'relax'
 local function scan_filename()
   local name = {}
@@ -116,40 +23,43 @@ local function scan_filename()
     tok = token.scan_token()
     cmd = tok.command
   until cmd ~= spacer_cmd and cmd ~= relax_cmd
-  while (tok.command <= 12 and tok.mode <= token.biggest_char()
+  while (tok.command <= 12 and tok.command > 0 and tok.command ~= 9 and tok.index <= token.biggest_char()
           or (token.put_next(tok) and false))
-      and (quoted or tok.mode ~= string.byte' ') do
-    if tok.mode == string.byte'"' then
+      and (quoted or tok.index ~= string.byte' ') do
+    if tok.index == string.byte'"' then
       quoted = not quoted
     else
-      name[#name+1] = tok.mode
+      name[#name+1] = tok.index
     end
     tok = token.scan_token()
   end
   return utf8.char(table.unpack(name))
 end
 
+local l = lpeg or require'lpeg'
+local add_file_extension = l.Cs((1-('.' * (1-l.S'./\\')^0) * -1)^0 * (l.P(1)^1+l.Cc'.tex'))
 local ofiles = {}
 local function do_openout(p)
   if ofiles[p.file] then
     error[[Existing file]]
   else
     local msg
-    ofiles[p.file], msg = io.open(p.name, 'w')
+    ofiles[p.file], msg = io.open(add_file_extension:match(p.name), 'w')
     if not ofiles[p.file] then
       error(msg)
     end
   end
 end
+local open_whatsit = new_whatsit('open', do_openout)
 token.luacmd("openout", function(_, immediate) -- \openout
   local file = token.scan_int()
   token.scan_keyword'='
   local name = scan_filename()
-  local props = {file = file, name = name, handle = do_openout}
+  local props = {file = file, name = name}
   if immediate == "immediate" then
     do_openout(props)
   else
-    local whatsit = node.direct.new(whatsit_id, whatsits.open)
+    local whatsit = node.direct.new(whatsit_id, open_whatsit)
     properties[whatsit] = props
     node.direct.write(whatsit)
   end
@@ -160,13 +70,14 @@ local function do_closeout(p)
     ofiles[p.file] = nil
   end
 end
+local close_whatsit = new_whatsit('close', do_closeout)
 token.luacmd("closeout", function(_, immediate) -- \closeout
   local file = token.scan_int()
-  local props = {file = file, handle = do_closeout}
+  local props = {file = file}
   if immediate == "immediate" then
     do_closeout(props)
   else
-    local whatsit = node.direct.new(whatsit_id, whatsits.close)
+    local whatsit = node.direct.new(whatsit_id, close_whatsit)
     properties[whatsit] = props
     node.direct.write(whatsit)
   end
@@ -180,47 +91,34 @@ local function do_write(p)
     texio.write_nl(p.file < 0 and "log" or "term and log", content)
   end
 end
+local write_whatsit = new_whatsit('write', do_write)
 token.luacmd("write", function(_, immediate) -- \write
   local file = token.scan_int()
   local content = token.scan_tokenlist()
-  local props = {file = file, data = content, handle = do_write}
+  local props = {file = file, data = content}
   if immediate == "immediate" then
     do_write(props)
   else
-    local whatsit = node.direct.new(whatsit_id, whatsits.write)
+    local whatsit = node.direct.new(whatsit_id, write_whatsit)
     properties[whatsit] = props
     node.direct.write(whatsit)
   end
 end, "protected")
 
+local functions = lua.get_functions_table()
 token.luacmd("immediate", function() -- \immediate
   local next_tok = token.scan_token()
   if next_tok.command ~= lua_call_cmd then
     return token.put_next(next_tok)
   end
-  local function_id = next_tok.mode
-  functions[function_id](function_id, 'immediate')
+  local function_id = next_tok.index
+  return functions[function_id](function_id, 'immediate')
 end, "protected")
 -- functions[43] = function() -- \pdfvariable
 --   local name = token.scan_string()
 --   print('Missing \\pdf' .. name)
 -- end
-if status.ini_version then
-  function fixupluafunctions()
-    return predefined_luafunctions
-  end
-else
-  function fixupluafunctions(i)
-    predefined_luafunctions = i
-  end
-  local prepared_code = lua.bytecode[1]
-  prepared_code()
-  lua.bytecode[1] = nil
-  -- function fixupluafunctions()
-    -- new_luafunction = luatexbase.new_luafunction
-    -- fixupluafunctions = nil
-  -- end
-end
+require'luametalatex-baseregisters'
 require'luametalatex-back-pdf'
 require'luametalatex-node-luaotfload'
 
@@ -243,4 +141,3 @@ token.luacmd("Umathcodenum", function(_, scanning)
     tex.setmathcodes(char, (mathcode >> 21) & 7, mathcode >> 24, mathcode & 0x1FFFFF)
   end
 end, "force", "global", "value")
-

luametalatex-font-pk.lua

@@ -0,0 +1,203 @@
+local function decode_char(buf)
+  local flag = buf:byte(1)
+  local form = flag & 0x07
+  local tfm, dx, dy, w, h, hoff, voff, off
+  if form < 4 then
+    tfm, dx, w, h, hoff, voff, off = string.unpack(">I3BBBbb", buf, 4)
+    dx, dy = dx * 2^16, 0
+  elseif form < 7 then
+    tfm, dx, w, h, hoff, voff, off = string.unpack(">I3HHHhh", buf, 5)
+    dx, dy = dx * 2^16, 0
+  else
+    tfm, dx, dy, w, h, hoff, voff, off = string.unpack(">I4I4I4I4I4i4i4", buf, 10)
+  end
+  local dyn_f, state = flag >> 4, flag & 8 == 8
+  local data
+  local stride = w+7>>3
+  if dyn_f == 14 then
+    print(state)
+    -- assert(not state)
+    data = lua.newtable(stride*h, 0)
+    local bit_offset, saved = 0
+    local delta_bit_offset = 8 - w%8
+    delta_bit_offset = delta_bit_offset == 8 and 0 or delta_bit_offset
+    for y=0,h-1 do
+      for x=1,stride do
+        if bit_offset == 0 then
+          saved = buf:byte(off)
+          data[y*stride+x] = saved
+        else
+          local saved_mask = (1<<bit_offset)-1
+          local current = (saved&saved_mask) << 8-bit_offset
+          saved = buf:byte(off)
+          data[y*stride+x] = current | (saved & ~saved_mask) >> bit_offset
+        end
+        off = off+1
+      end
+      if delta_bit_offset then
+        data[(y+1)*stride] = data[(y+1)*stride] & (0x100 - (1<<delta_bit_offset))
+      end
+      bit_offset = bit_offset + delta_bit_offset
+      if bit_offset >= 8 then
+        bit_offset = bit_offset-8
+        off = off-1
+        saved = buf:byte(off)
+      end
+    end
+  else
+    data = {string.rep('\0', stride*h):byte(1,-1)} -- FIXME: This is probably really slow
+    local nibble, repeat_row = nil, 0
+    local function get_nibble()
+      if nibble then
+        local cur = nibble
+        nibble = nil
+        off = off+1
+        return cur
+      else
+        local cur = buf:byte(off)
+        nibble = cur&0xF
+        return cur >> 4
+      end
+    end
+    local function get_packed()
+      local cur = get_nibble()
+      if cur == 0 then
+        local i = 0
+        repeat
+          cur = get_nibble()
+          i = i+1
+        until cur ~= 0
+        for _=1,i do
+          cur = (cur<<4) + get_nibble()
+        end
+        return cur - 0xF + (13-dyn_f << 4) + dyn_f
+      elseif cur <= dyn_f then
+        return cur
+      elseif cur < 14 then
+        return (cur-dyn_f-1 << 4) + get_nibble() + dyn_f + 1
+      else
+        repeat_row = cur == 14 and get_packed() or 1
+        return get_packed()
+      end
+    end
+    local cur_x, cur_y = 0, 0
+    while cur_y < h do
+      local count = get_packed()
+      repeat
+        local this_line = math.min(w - cur_x, count)
+        count = count - this_line
+        if state then
+          local cur_bit_offset = cur_x % 8
+          if cur_bit_offset ~= 0 then -- We are in the middle of a byte
+            cur_bit_offset = 8-cur_bit_offset -- The remaining bits in the byte
+            local off = cur_y*stride+(cur_x>>3)+1
+            if this_line > cur_bit_offset then -- Fill byte with ones
+              data[off] = data[off] + (1<<cur_bit_offset)-1
+              this_line, cur_x = this_line-cur_bit_offset, cur_x+cur_bit_offset
+            else
+              data[off] = data[off] + (1<<cur_bit_offset)-(1<<cur_bit_offset-this_line)
+              this_line, cur_x = 0, cur_x+this_line
+            end
+          end
+          while this_line >= 8 do
+            data[cur_y*stride+(cur_x>>3)+1] = 0xFF
+            this_line, cur_x = this_line-8, cur_x+8
+          end
+          if this_line ~= 0 then
+            data[cur_y*stride+(cur_x>>3)+1] = 0x100-(1<<8-this_line)
+          end
+        end
+        cur_x = cur_x + this_line
+        if cur_x == w then
+          for i = 1, repeat_row do
+            table.move(data, cur_y*stride+1, (cur_y+1)*stride, (cur_y+i)*stride+1) -- TODO
+          end
+          cur_y, cur_x, repeat_row = cur_y + 1 + repeat_row, 0, 0
+        end
+      until count == 0
+      state = not state
+    end
+  end
+  data = string.char(table.unpack(data))
+  return {
+    data = data,
+    tfm = tfm,
+    dx = dx,
+    dy = dy,
+    hoff = hoff,
+    voff = voff,
+    w = w,
+    h = h,
+  }
+end
+
+local commands = {
+  [240] = function(buf, off, t)
+    local xxx xxx, off = string.unpack(">xs1", buf, off)
+    return off
+  end,
+  [241] = function(buf, off, t)
+    local xxx xxx, off = string.unpack(">xs2", buf, off)
+    return off
+  end,
+  [242] = function(buf, off, t)
+    local xxx xxx, off = string.unpack(">xs3", buf, off)
+    return off
+  end,
+  [243] = function(buf, off, t)
+    local xxx xxx, off = string.unpack(">xs4", buf, off)
+    return off
+  end,
+  [244] = function(buf, off, t)
+    local yyy yyy, off = string.unpack(">xI4", buf, off)
+    return off
+  end,
+  [247] = function(buf, off, t)
+    local ident
+    ident, t.comment, t.designsize, t.checksum, t.hppp, t.vppp, off = string.unpack(">xBs1I4I4I4I4", buf, off)
+    if ident ~= 89 then
+      error[[Not a PK file]]
+    end
+    return off
+  end,
+}
+local function parse_commands(buf, off, t)
+  local cmd = buf:byte(off)
+  assert(cmd == 247)
+  repeat
+    if cmd < 240 then
+      local form = cmd & 0x07
+      local chr, newoff, length
+      if form < 4 then
+        length, chr, newoff = string.unpack(">xBB", buf, off)
+        length = length + ((form & 3)<<8)
+      elseif form < 7 then
+        length, chr, newoff = string.unpack(">xHB", buf, off)
+        length = length + ((form & 3)<<16)
+      else
+        length, chr, newoff = string.unpack(">xI4I4", buf, off)
+      end
+      newoff = newoff + length
+      t[chr] = decode_char(buf:sub(off, newoff))
+      off = newoff
+    else
+      local handler = commands[cmd]
+      if not handler then
+        print([[Unknown command ]] .. cmd)
+        return off-1
+      end
+      off = handler(buf, off, t)
+    end
+    cmd = buf:byte(off)
+  until cmd == 245
+  return off
+end
+return function(filename)
+  local f = assert(io.open(filename, 'rb'))
+  local pk = f:read'a'
+  f:close()
+  local res = {}
+  local off = parse_commands(pk, 1, res)
+  -- assert(off == #pk+1) -- TODO: Check that only fillup bytes follow
+  return res
+end

luametalatex-font-resolve.lua

@@ -0,0 +1,167 @@
+font.read_tfm = require'luametalatex-font-tfm'
+local read_vf = require'luametalatex-font-vf'
+font.read_vf = read_vf
+local fontmap = require'luametalatex-pdf-font-map'.fontmap
+local callback_find = callback.find
+
+local old_font_define = font.define
+local old_addcharacters = font.addcharacters
+
+require'luametalatex-pdf-font-map'.mapfile(kpse.find_file('pdftex.map', 'map', true))
+
+local all_fonts = {}
+font.fonts = all_fonts
+function font.getfont(id)
+  return all_fonts[id]
+end
+
+local fontextensions = {
+  ttf = {"truetype", "truetype fonts",},
+  otf = {"opentype", "opentype fonts",},
+  pfb = {"type1", "type1 fonts",},
+}
+fontextensions.cff = fontextensions.otf
+local fontformats = {
+  fontextensions.pfb, fontextensions.otf, fontextensions.ttf,
+}
+
+local special_parser do
+  local l = lpeg or require'lpeg'
+  local space = l.S' '^0
+  local name = (1-l.P' ')^1
+  local reencode = name * space * 'ReEncodeFont'
+  local digit = l.R'09'
+  local number = digit^1 * ('.' * digit^0) + '.' * digit^1/tonumber
+  local milli_stmt = number * space * ('SlantFont' * l.Cc'slant' + 'ExtendFont' * l.Cc'extend') / function(n, k)
+    return k, (n*1000 + .5)//1
+  end
+  special_parser = l.Cf(l.Carg(1) * (space * (reencode + milli_stmt))^0 * space * -1, rawset)
+end
+
+function font.define(f)
+  if (f.type or "unknown") == "unknown" then
+    local vf = read_vf(f.name, f.size)
+    if vf then
+      f.type = 'virtual'
+      f.fonts = vf.fonts
+      local realchars = f.characters
+      for cp, char in next, vf.characters do
+        assert(realchars[cp]).commands = char.commands
+      end
+    else
+      f.type = 'real'
+    end
+  end
+  local format = f.format or "unknown"
+  local encodingbytes = f.encodingbytes or (f.format:sub(5) == "type" and 2 or 1)
+  f.encodingbytes = encodingbytes
+  if encodingbytes == 1 and f.type ~= 'virtual' and f.format ~= 'type3node' then
+    -- map file lookup
+    local entry = fontmap[f.name]
+    if entry then
+      local filename = entry[3]
+      local format = filename and filename:sub(-4, -4) == '.' and fontextensions[filename:sub(-3, -1)]
+      if format then
+        f.format = format[1]
+        f.filename = kpse.find_file(filename, format[2])
+        local encoding = entry[4]
+        if encoding then
+          f.encoding = kpse.find_file(encoding, 'enc files')
+        end
+        if entry[5] then
+          assert(special_parser:match(entry[5], 1, f))
+        end
+      else
+        local done = false
+        for _, format in ipairs(fontformats) do
+          local filename = kpse.find_file(filename, format[2])
+          if filename then
+            f.filename = filename
+            f.format = format[1]
+            local encoding = entry[4]
+            if encoding then
+              f.encoding = kpse.find_file(encoding, 'enc files')
+            end
+            if entry[5] then
+              assert(special_parser:match(entry[5], 1, f))
+            end
+            done = true
+            break
+          end
+        end
+        if not done then
+          print('!', 'type3', require'inspect'(entry))
+          f.format = "type3"
+        end
+      end
+    else
+      f.format = "type3"
+    end
+  end
+  local id = old_font_define(f)
+  all_fonts[id] = f
+  if f.fonts then
+    for i, f in next, f.fonts do
+      if not f.id then
+        f.id = assert(callback_find'define_font'(f.name, f.size or -1000))
+      elseif f.id == 0 then
+        f.id = id
+      end
+    end
+  end
+  return id
+end
+
+function font.addcharacters(fid, newdir)
+  old_addcharacters(fid, newdir) -- The easy part, the remaining stuff gets crazy
+  local fontdir = assert(all_fonts[fid], 'addcharacters expects an existing font')
+  local fonts_map
+  if newdir.fonts then -- FIXME: Handle default fonts table
+    if fontdir.fonts then
+      fonts_map = {}
+      for i,f in next, newdir.fonts do
+        if not f.id then
+          f.id = assert(callback_find'define_font'(f.name, f.size or -1000))
+        elseif f.id == 0 then
+          f.id = fid
+        end
+        for j,ff in next, fontdir.fonts do
+          if ff.id == f.id then
+            fonts_map[i] = j
+            goto FONT_MAPPING_SET -- A typical for ... do ... else ... end implemented using goto.
+          end
+        end
+        -- NOT FOUND, so add it
+        local new_f = #fontdir.fonts + 1
+        fontdir.fonts[new_f] = f
+        fonts_map[i] = f
+        ::FONT_MAPPING_SET::
+      end
+    else
+      fontdir.fonts = newdir.fonts
+    end
+  end
+  for cp, glyph in next, newdir do
+    local existing = fontdir[cp]
+    if existing ~= glyph then
+      if existing then
+        -- texio.write_nl'Overwriting existing character. Here be dragons'
+      end
+    end
+    if glyph.commands then
+      local font_seen
+      for _, cmd in ipairs(glyph.commands) do
+        if cmd[1] == 'font' then
+          font_seen = true
+          cmd[2] = fonts_map[cmd[2]]
+        elseif cmd[1] == 'slot' then
+          font_seen = true
+          cmd[2] = fonts_map[cmd[2]]
+        elseif not font_seen and cmd[1] == 'char' then
+          font_seen = true
+          cmd[1], cmd[2], cmd[3] = 'slot', fonts_map[1], cmd[2]
+        end
+      end
+    end
+  end
+end

luametalatex-font-t1.lua

@@ -44,7 +44,7 @@ end
 local boolean = (lpeg.P'true' + 'false')/{["true"] = true, ["false"] = false}
 local anytype = {hexstring + literalstring + number + lname + boolean + lpeg.V(2) + name, lpeg.Ct('[' * (white^-1 * lpeg.V(1))^0 * white^-1 * ']' + '{' * (white^-1 * lpeg.V(1))^0 * white^-1 * '}' * white^-1 * lpeg.P"executeonly"^-1)}
 local dict = lpeg.Cf(lpeg.Carg(1) * lpeg.Cg(white^-1*lname*white^-1*(anytype)*white^-1*lpeg.P"readonly"^-1*white^-1*lpeg.P"noaccess"^-1*white^-1*(lpeg.P"def"+"ND"+"|-"))^0, rawset)
-local encoding = (white+anytype-("for"*white))^0*"for"*white/0
+local encoding = (white+anytype-("dup"*white))^0/0
                * lpeg.Cf(lpeg.Ct''
                  * lpeg.Cg("dup"*white*number*white^-1*lname*white^-1*"put"*white)^0
                  , rawset)
@@ -63,7 +63,7 @@ local function parse_fontinfo(offset, str)
   offset = (white^-1*"end"*white^-1*lpeg.P"readonly"^-1*white^-1*"def"):match(str, offset)
   return found, offset
 end
-local binary_bytes = lpeg.Cmt(number*white^-1*(lpeg.P'-| ' + 'RD '), function(s, p, l)return p+l, s:sub(p, p+l-1) end)*white*(lpeg.P"|-"+"|"+"ND"+"NP")
+local binary_bytes = lpeg.Cmt(number*white^-1*(lpeg.P'-| ' + 'RD '), function(s, p, l)return p+l, s:sub(p, p+l-1) end)*white^-1*(lpeg.P"|-"+"|"+"ND"+"NP")
 local charstr = white^-1*lname*(white^-1*(anytype-lname))^0/0*white^-1
             * lpeg.Cf(lpeg.Ct''
               * lpeg.Cg(lname*white^-1*binary_bytes*white)^0

luametalatex-font-tfm.lua

@@ -15,6 +15,27 @@ local function read_scaled(buf, i, count, factor)
   end
   return result,  i + count * 4
 end
+local function parse_ligkern(buf, offset, r_boundary, kerns)
+  local kerning, ligatures, done = {}, {}, {}
+  repeat
+    local skip, next, op, rem
+    skip, next, op, rem, offset = string.unpack("BBBB", buf, offset)
+    if skip > 128 then break end
+    if next == r_boundary then next = "right_boundary" end
+    if not done[next] then
+      done[next] = true
+      if op >= 128 then
+        kerning[next] = kerns[(op - 128 << 8) + rem + 1]
+      else
+        ligatures[next] = {
+          type = op,
+          char = rem,
+        }
+      end
+    end
+  until skip == 128
+  return next(kerning) and kerning or nil, next(ligatures) and ligatures or nil
+end
 local function parse_tfm(buf, i, size)
   local lf, lh, bc, ec, nw, nh, nd, ni, nl, nk, ne, np
   lf, lh, bc, ec, nw, nh, nd, ni, nl, nk, ne, np, i =
@@ -43,7 +64,7 @@ local function parse_tfm(buf, i, size)
     italics, i = read_scaled(buf, i, ni, size)
     for k,v in ipairs(italics) do if v == 0 then italics[k] = nil end end
     ligatureoffset = i
-    if string.byte(buf, i, i) > 128 then
+    if nl ~= 0 and string.byte(buf, i, i) == 255 then
       r_boundary = string.byte(buf, i+1, i+1)
     end
     i = i + nl * 4
@@ -56,7 +77,7 @@ local function parse_tfm(buf, i, size)
       end
       extensibles[j] = ext
     end
-    local slant = string.unpack(">i4", buf, i) >> 4
+    local slant = np ~= 0 and string.unpack(">i4", buf, i) >> 4 or nil
     parameters = read_scaled(buf, i, np, size)
     parameters[1] = slant
   end
@@ -76,30 +97,9 @@ local function parse_tfm(buf, i, size)
       elseif tag == 1 then
         local offset = (charinfo & 0xFF) * 4 + ligatureoffset
         if string.byte(buf, offset, offset) > 128 then
-          offset = string.unpack(">H", buf, offset + 2)
+          offset = string.unpack(">H", buf, offset + 2) * 4 + ligatureoffset
         end
-        char.kerns, char.ligatures = {}, {}
-        local done = {}
-        repeat
-          local skip, next, op, rem
-          skip, next, op, rem, offset = string.unpack("BBBB", buf, offset)
-          if skip > 128 then break end
-          if next == r_boundary then next = "right_boundary" end
-          if not done[next] then
-            done[next] = true
-            if op >= 128 then
-              char.kerns[next] = kerns[(op - 128 << 8) + rem + 1]
-            else
-              char.ligatures[next] = {
-                type = op,
-                char = rem,
-              }
-            end
-          end
-          offset = offset + 4*skip
-        until skip == 128
-        if not next(char.kerns) then char.kerns = nil end
-        if not next(char.ligatures) then char.ligatures = nil end
+        char.kerns, char.ligatures = parse_ligkern(buf, offset, r_boundary, kerns)
       elseif tag == 2 then
         char.next = charinfo & 0xFF
       elseif tag == 3 then
@@ -108,6 +108,12 @@ local function parse_tfm(buf, i, size)
       characters[cc] = char
     end
   end
+  if nl ~= 0 and string.byte(buf, ligatureoffset + (nl-1) * 4) == 255 then
+    local char = {}
+    characters.left_boundary = char
+    local offset = string.unpack(">H", buf, ligatureoffset + nl * 4 - 2) * 4 + ligatureoffset
+    char.kerns, char.ligatures = parse_ligkern(buf, offset, r_boundary, kerns)
+  end
   return {
     checksum = checksum,
     direction = 0,
@@ -136,6 +142,7 @@ end
 local basename = ((1-lpeg.S'\\/')^0*lpeg.S'\\/')^0*lpeg.C((1-lpeg.P'.tfm'*-1)^0)
 return function(name, size)
   local filename = kpse.find_file(name, 'tfm', true)
+  if not filename then return end
   local f = io.open(filename)
   if not f then return end
   local buf = f:read'*a'

luametalatex-font-vf.lua

@@ -9,10 +9,11 @@ local function read_fonts(buf, i, fonts, size)
   if not cmd then return i end
   local fid, check, scale, designsize, arealen, namelen, i =
     string.unpack(cmd, buf, i + 1)
+  fid = fid + 1 -- We prefer 1-based arrays
   local fsize = size * scale >> 20
   if fonts[fid] then error[[font number reused in VF file]] end
   fonts[fid] = {
-    area = string.sub(buf, i, i+arealen-1),
+    area = arealen > 0 and string.sub(buf, i, i+arealen-1) or nil,
     name = string.sub(buf, i+arealen, i+arealen+namelen-1),
     size = fsize,
     designsize = designsize >> 4,
@@ -134,9 +135,9 @@ local function read_chars(buf, i, characters, size)
       if cmd >= 235 then
         cmd, i = string.unpack(Cmds[cmd-234], buf, i + 1)
       else
-        i = i + 1
+        cmd, i = cmd - 171, i + 1
       end
-      commands[#commands + 1] = { "font", cmd }
+      commands[#commands + 1] = { "font", cmd + 1 } -- 1-based fonts
     elseif xxx[cmd] then
       cmd, i = string.unpack(xxx[cmd], buf, i + 1)
       commands[#commands + 1] = { "special", cmd }
@@ -161,16 +162,17 @@ local function parse_vf(buf, i, size)
   i = read_fonts(buf, i, fonts, size)
   i = read_chars(buf, i, characters, size)
 
-  print(require'inspect'(font))
+  return font
 end
 local basename = ((1-lpeg.S'\\/')^0*lpeg.S'\\/')^0*lpeg.C((1-lpeg.P'.tfm'*-1)^0)
 return function(name, size, must_exist)
   local filename = kpse.find_file(name, 'vf', must_exist)
+  if not filename then return end
   local f = io.open(filename)
   if not f then return end
   local buf = f:read'*a'
   f:close()
-  local result = parse_tfm(buf, 1, size)
+  local result = parse_vf(buf, 1, size)
   result.name = basename:match(name)
   return result
 end

luametalatex-init.lua

@@ -1,7 +1,8 @@
 do
-  local ourpath = arg[0]:match('^%-%-lua=(.*)luametalatex%-init%.lua$')
+  local ourpath = lua.startupfile:match('(.*[/\\])[^/\\]*%.lua$')
   kpse = assert(package.loadlib(ourpath .. 'kpse.so', 'luaopen_kpse'))()
 end
+local interaction
 do
   local arg0, progname
   for _, a in ipairs(arg) do
@@ -9,8 +10,20 @@ do
       progname = a:sub(12)
     elseif a:sub(1,7) == "--arg0=" then
       arg0 = a:sub(8)
+    elseif a:match'^%-%-?interaction=' then
+      local interaction_name = a:sub(a:find'='+1)
+      interaction = ({
+        batchmode=0,
+        nonstopmode=1,
+        scrollmode=2,
+        errorstopmode=3,
+      })[interaction_name]
+      if not interaction then
+        texio.write('term', string.format('Unknown interaction mode %q ignored.\n', interaction_name))
+      end
     end
   end
+  os.arg0 = arg0
   kpse.set_program_name(arg0, progname)
 end
 package.searchers[2] = function(modname)
@@ -24,100 +37,36 @@ package.searchers[2] = function(modname)
   end
   return mod, filename
 end
--- kpse.set_maketex("kpse_fmt_format", true)
-bit32 = require'luametalatex-bit32'
-kpse.init_prog("LUATEX", 400, "nexthi", nil)
-status.init_kpse = 1
+kpse.set_maketex("fmt", true, "compile")
+kpse.set_maketex("pk", true, "compile")
 require'luametalatex-init-config'
-status.safer_option = 0
-status.shell_escape = 0
-local read_tfm = require'luametalatex-font-tfm'
-local read_vf = require'luametalatex-font-vf'
-font.read_tfm = read_tfm
-font.read_vf = read_vf
-require'module'
-font.fonts = {}
-function font.getfont(id)
-  return font.fonts[id]
-end
-pdf = {
-  getfontname = function(id) -- No font sharing
-    return id
-  end,
-}
-local olddefinefont = font.define
-function font.define(f)
-  local i = olddefinefont(f)
-  font.fonts[i] = f
-  return i
-end
-callback.register('define_font', function(name, size)
-  local f = read_tfm(name, size)
-  local id = font.define(f)
-  if status.ini_version then
-    lua.prepared_code[#lua.prepared_code+1] = string.format("assert(%i == font.define(font.read_tfm(%q, %i)))", id, name, size)
-  end
-  return id
-end)
-callback.register('find_log_file', function(name) return name end)
--- callback.register('find_read_file', function(i, name) return kpse.find_file(name, 'tex', true) end)
-callback.register('find_data_file', function(name, ...) return kpse.find_file(name, 'tex', true) end)
-callback.register('read_data_file', function(name) error[[TODO]]return kpse.find_file(name, 'tex', true) end)
--- local file_meta = {\
-callback.register('open_data_file', function(name)
-  local f = io.open(name)
-  return setmetatable({
-    reader = function() return f:read() end,
-    close = function()error[[1]] return f:close() end,
-  }, {
-    __gc = function()f:close()end,
-  })
-end)
-callback.register('find_format_file', function(name) return kpse.find_file(name, 'fmt', true) end)
-callback.register('handle_error_hook', function()
-  repeat
-    texio.write_nl'? '
-    local line = io.read()
-    if not line then
-      error[[TODO: Handle EOL]]
-    end
-    if line == "" then return 3 end
-    local first = line:sub(1,1):upper()
-    if first == 'H' then
-      texio.write(tex.gethelptext())
-    elseif first == 'I' then
-      line = line:sub(2)
-      tex.runtoks(function()
-        tex.sprint(token.scan_token(), line)
-      end)
-      return 3
-    elseif first == 'Q' then texio.write'OK, entering \\batchmode...\n' return 0
-    elseif first == 'R' then texio.write'OK, entering \\nonstopmode...\n' return 1
-    elseif first == 'S' then texio.write'OK, entering \\scrollmode...\n' return 2
-    elseif first == 'X' then return -1
-    else
-      texio.write'Type <return> to proceed, S to scroll future error messages,\
-\z      R to run without stopping, Q to run quietly,\
-\z      I to insert something,\
-\z      H for help, X to quit.'
-    end
-  until false
-  -- print('handle')
-  return 3
-end)
-callback.register('pre_dump', function()
-  lua.prepared_code[1] = string.format("fixupluafunctions(%i)", fixupluafunctions())
-  lua.bytecode[1] = assert(load(table.concat(lua.prepared_code, ' ')))
-end)
-function texconfig.init()
-  lua.bytecode[2]()
-  if not status.ini_version then
-    lua.bytecode[2] = nil
-  end
-end
+local callback_register = callback.register
+local build_bytecode
 if status.ini_version then
-  lua.prepared_code = {false}
-  local code = package.searchers[2]('luametalatex-firstcode')
-  if type(code) == "string" then error(string.format("Initialization code not found %s", code)) end
-  lua.bytecode[2] = code
+  local build_bytecode_mod = require'luametalatex-build-bytecode'
+  local preloaded_modules = {}
+  local old_searcher = package.searchers[2]
+  package.searchers[2] = function(name)
+    local mod, file = old_searcher(name)
+    if not file then return mod end -- Only works because we always return file when successful
+    preloaded_modules[#preloaded_modules+1] = {name, file}
+    return mod, file
+  end
+  function build_bytecode(str)
+    return load(build_bytecode_mod(preloaded_modules) .. "\nrequire'luametalatex-lateinit'(function()" .. str .. '\nend)', 'preloaded', 't')
+  end
+end
+
+callback_register('find_format_file', function(name) return kpse.find_file(name, 'fmt', true) end)
+function texconfig.init()
+  if interaction then
+    tex.setinteraction(interaction)
+  end
+  if build_bytecode then -- Effectivly if status.ini_version
+    require'luametalatex-lateinit'(build_bytecode)
+  else
+    local register = tex.count[262]+1
+    lua.bytecode[register]()
+    lua.bytecode[register] = nil
+  end
 end

luametalatex-lateinit.lua

@@ -0,0 +1,127 @@
+local initex = status.ini_version
+
+if initex then
+  lua.prepared_code = {false}
+end
+
+bit32 = require'luametalatex-bit32' -- Why? And why so early?
+status.init_kpse = 1 -- Why?
+status.safer_option = 0 -- compat
+status.shell_escape = 0 -- compat -- This is currently a lie.
+-- require'module' -- ???
+pdf = {
+  variable = {},
+}
+require'luametalatex-font-resolve' -- Replace font.define. Must be loaded before callbacks
+require'luametalatex-callbacks'
+
+local functions = lua.getfunctionstable()
+-- I am not sure why this is necessary, but otherwise LuaMetaTeX resets
+-- the functions table every time the getter is called
+function lua.get_functions_table() return functions end
+local set_lua = token.set_lua
+-- There are two approaches to manage luafunctions ids without triggering
+-- issues with ltluatex assigned numbers: Originally we assigned numbers
+-- starting with 1, then switched to luatexbase ASAP and synchronised both
+-- numbering schemes. But there is a useful quirk in ltluatex's luafunction
+-- allocator: It only uses numbers upto 65535, so we can just use bigger
+-- numbers. (This might have negative repercussins on performance because it
+-- probably doesn't store the function in the array part of the Lua table.
+-- Let's reconsider if this ever becomes a problem.
+-- local new_luafunction = luatexbase.new_luafunction
+local predefined_luafunctions = initex and 65536 -- 1<<16  -- We start with 1<<16 + 1 (1<<16=65536 is reserved for luametalatex-local)
+local new_luafunction
+function new_luafunction(name)
+  if predefined_luafunctions then
+    predefined_luafunctions = predefined_luafunctions + 1
+    return predefined_luafunctions
+  else
+    error[[Here only preexisting luafunctions should be set]]
+  end
+end
+local undefined_cmd = token.command_id'undefined_cs'
+local lua_call_cmd = token.command_id'lua_call'
+local lua_value_cmd = token.command_id'lua_value'
+local lua_expandable_call_cmd = token.command_id'lua_expandable_call'
+function token.luacmd(name, func, ...)
+  local idx
+  local tok = token.create(name)
+  local cmd = tok.command
+  if cmd == lua_value_cmd then
+    idx = tok.index
+  elseif cmd == lua_call_cmd then
+    idx = tok.index
+  elseif cmd == lua_expandable_call_cmd then
+    idx = tok.index
+  elseif ... == 'force' then
+    idx = new_luafunction(name)
+    set_lua(name, idx, select(2, ...))
+  elseif cmd == undefined_cmd then
+    idx = new_luafunction(name)
+    set_lua(name, idx, ...)
+  else
+    error(tok.cmdname)
+  end
+  if functions[idx] then
+    error[[Already defined]]
+  end
+  functions[idx] = func
+  return idx
+end
+
+if initex then
+  local build_bytecode = nil -- To be filled
+  callback.register('pre_dump', function()
+    local prepared = lua.prepared_code
+    prepared[1] = string.format("fixupluafunctions(%i)", predefined_luafunctions)
+    for i=0,0 do -- maybeFIXME: In practise only one language is preloaded in LuaTeX anyway
+    -- for i=0,tex.count[19] do -- Sometimes catches reserved language ids which are not used yet
+    -- for i=0,lang.new():id()-1 do -- lang.new():id() is always 0 in luametatex?!?
+      local l = lang.new(i)
+      local str = string.format("do \z
+        local l = lang.new(%i)\z
+        l:hyphenationmin(%i)\z
+        l:prehyphenchar(%i)\z
+        l:posthyphenchar(%i)\z
+        l:preexhyphenchar(%i)\z
+        l:postexhyphenchar(%i)",
+        i,
+        l:hyphenationmin(),
+        l:prehyphenchar(),
+        l:posthyphenchar(),
+        l:preexhyphenchar(),
+        l:postexhyphenchar())
+      local patterns = l:patterns()
+      local exceptions = l:hyphenation()
+      if patterns and exceptions then
+        str = string.format("%sl:patterns(%q)l:hyphenation(%q)end", str, patterns, exceptions)
+      elseif patterns then
+        str = string.format("%sl:patterns(%q)end", str, patterns)
+      elseif exceptions then
+        str = string.format("%sl:hyphenation(%q)end", str, exceptions)
+      else
+        str = str .. 'end'
+      end
+      prepared[#prepared+1] = str
+    end
+    for i=2,#prepared do
+      if type(prepared[i]) ~= 'string' then
+        prepared[i] = assert(prepared[i]())
+      end
+    end
+    lua.bytecode[tex.count[262]+1] = build_bytecode(table.concat(prepared, '\n'))
+  end)
+  return function(f)
+    build_bytecode = f
+    return require'luametalatex-firstcode'
+  end
+else
+  function fixupluafunctions(i)
+    predefined_luafunctions = i
+    fixupluafunctions = nil
+  end
+  return function(f)
+    f()
+    return require'luametalatex-firstcode'
+  end
+end

luametalatex-local.lua

@@ -0,0 +1,52 @@
+-- Implement support for local variables.
+
+local stack = {}
+
+local restore_func = 65536 -- Reserved in firstcode
+lua.get_functions_table()[restore_func] = function()
+  local level = assert(stack[tex.currentgrouplevel], 'Out of sync')
+  stack[tex.currentgrouplevel] = nil
+  for t,entries in next, level do
+    for k,v in next, entries do
+      t[k] = v
+    end
+  end
+end
+local restore_toks = {token.new(2, token.command_id'after_something') , token.new(restore_func, token.command_id'lua_call')} -- \atendofgroup FIXME: Detect mode automatically once token.primitive is fixed
+local put_next = token.put_next
+local runtoks = tex.runtoks
+local function put_restore_toks()
+  put_next(restore_toks)
+end
+
+return function(t, k, v, global)
+  local l = tex.currentgrouplevel
+  if global then
+    for i=1,l do
+      local level = stack[i]
+      if level then
+        local saved = level[t]
+        if saved then
+          saved[k] = nil
+        end
+      end
+    end
+  elseif l > 0 then
+    local level = stack[l]
+    if not level then
+      level = {}
+      runtoks(put_restore_toks)
+      stack[l] = level
+    end
+
+    local saved = level[t]
+    if not saved then
+      saved = {}
+      level[t] = saved
+    end
+
+    saved[k] = saved[k] or t[k]
+  end
+
+  t[k] = v
+end

luametalatex-ltexpl-hook.tex

@@ -0,0 +1,7 @@
+\directlua{unhook_expl()}
+% See baseregisters for list of toks pdfvariables
+\directlua{initialize_pdf_toks()}
+\ifx\@tfor\undefined
+  \def\@tfor#1\do#2{}
+\fi
+\input ltexpl.ltx

luametalatex-microtype.sty

@@ -0,0 +1,34 @@
+% To be loaded *after* microtype
+%
+% Patching \pickup@font to emulate LuaTeX's font expansion interface using \adjustspacing...
+\RequirePackage{microtype}% Just to ensure it isn't loaded in the wrong position. Normally this shouldn't be necessary
+\def\luametalatex@@setexpansion#-#1#2#3#-\relax{%
+  \adjustspacingstretch #1
+  \adjustspacingshrink #2
+  \adjustspacingstep #3
+}%
+\newluafunction\luametalatex@@expandglyphsinfont
+\protected\luadef\pdffontexpand\luametalatex@@expandglyphsinfont
+\def\luametalatex@@everyjobandnow#1{\toksapp\everyjob{#1}#1}%
+%
+\begingroup
+\catcode`\!=\catcode`\%
+\catcode`\%=12
+\expanded{!
+  \endgroup
+  \def\noexpand\pickup@font{!
+    \unexpanded\expandafter{\pickup@font
+      \expandafter\expandafter\expandafter\luametalatex@@setexpansion\csname pickup@font@@hook@luametalatex@microtype@\the\fontid\font@name\endcsname{-1}{-1}{-1}\relax
+    }!
+  }!
+  \noexpand\luametalatex@@everyjobandnow{\noexpand\directlua{!
+    lua.get_functions_table()[\the\luametalatex@@expandglyphsinfont] = function()
+      token.put_next(token.create'fontid')
+      local font = token.scan_int()
+      local stretch = token.scan_int()
+      local shrink = token.scan_int()
+      local step = token.scan_int()
+      token.set_macro('pickup@font@@hook@luametalatex@microtype@' .. font, string.format("{}{%i}{%i}{%i}", stretch, shrink, step), "global")
+    end
+  }}
+}%

luametalatex-node-luaotfload.lua

@@ -1,4 +1,4 @@
--- Provide enough compatibility function to make luaotfload happy
+-- Provide enough compatibility functions to make luaotfload happy
 
 local properties = node.direct.get_properties_table()
 local flush = node.direct.flush_list
@@ -17,7 +17,7 @@ function node.direct.setcomponents(n, comp)
     properties[n] = props
     return
   end
-  local props_comp = props.components
+  local props_comp = rawget(props, 'components')
   if props_comp then
     props_comp.components = comp -- Important even if nil to avoid double-free
     if not comp then props.components = nil end

luametalatex-nodewriter.lua

@@ -29,6 +29,15 @@ local getexpansion = direct.getexpansion
 local getchar = direct.getchar
 local rangedimensions = direct.rangedimensions
 local traverse_id = direct.traverse_id
+local getdata = direct.getdata
+
+local utils = require'luametalatex-pdf-utils'
+local strip_floats = utils.strip_floats
+local to_bp = utils.to_bp
+
+local make_resources = require'luametalatex-pdf-resources'
+local pdf_font_map = require'luametalatex-pdf-font-deduplicate'
+local get_whatsit_handler = require'luametalatex-whatsits'.handler
 
 local dir_id = node.id'dir'
 
@@ -47,7 +56,6 @@ local function doublekeyed(t, id2name, name2id, index)
 end
 local nodehandler = (function()
   local function unknown_handler(_, n, x, y)
-    print(node.type(10))
     write(format("Sorry, but the PDF backend does not support %q (id = %i) nodes right now. The supplied node will be dropped at coordinates (%i, %i).", node.type(getid(n)), getid(n), x//1, y//1))
   end
   return doublekeyed({}, node.type, node.id, function()
@@ -69,14 +77,12 @@ local whatsithandler = (function()
   end)
 end)()
 local glyph, text, page, cm_pending = 1, 2, 3, 4
-local gsub = string.gsub
+
 local function projected_point(m, x, y, w)
   w = w or 1
   return x*m[1] + y*m[3] + w*m[5], x*m[2] + y*m[4] + w*m[6]
 end
-local function sp2bp(sp)
-  return sp/65781.76
-end
+local fontnames = setmetatable({}, {__index = function(t, k) local res = format("F%i", k) t[k] = res return res end})
 local topage
 local function totext(p, fid)
   local last = p.mode
@@ -90,15 +96,30 @@ local function totext(p, fid)
   p.mode = text
   if last == text and p.font.fid == fid then return end
   local f = font.getfont(fid) or font.fonts[fid]
-  if last ~= text then p.strings[#p.strings+1] = "BT" p.pos.lx, p.pos.ly, p.pos.x, p.pos.y, p.font.exfactor = 0, 0, 0, 0, 0 end
-  p:fontprovider(f, fid)
-  -- p.strings[#p.strings+1] = format("/F%i %f Tf 0 Tr", f.parent, sp2bp(f.size)) -- TODO: Setting the mode, expansion, etc.
+  if last ~= text then p.strings[#p.strings+1] = "BT" p.pos.lx, p.pos.ly, p.pos.x, p.pos.y, p.font.exfactor, p.font.extend, p.font.squeeze, p.font.slant = 0, 0, 0, 0, 0, 1, 1, 0 end
+
+  local pdf_fid = pdf_font_map[fid]
+  p.resources.Font[fontnames[pdf_fid]] = p.fontdirs[pdf_fid]
+  p.strings[#p.strings+1] = strip_floats(format("/F%i %f Tf 0 Tr", pdf_fid, to_bp(f.size))) -- TODO: Setting the mode, width, etc.
+  p.font.usedglyphs = p.usedglyphs[pdf_fid]
+
   p.font.fid = fid
   p.font.font = f
-  return false -- Return true if we need a new textmatrix
+  local need_tm = false
+  if p.font.extend ~= (f.extend or 1000)/1000 then
+    p.font.extend = (f.extend or 1000)/1000
+    need_tm = true
+  end
+  if p.font.squeeze ~= (f.squeeze or 1000)/1000 then
+    p.font.squeeze = (f.squeeze or 1000)/1000
+    need_tm = true
+  end
+  if p.font.slant ~= (f.slant or 0)/1000 then
+    p.font.slant = (f.slant or 0)/1000
+    need_tm = true
+  end
+  return need_tm
 end
-local inspect = require'inspect'
-local function show(t) print(inspect(t)) end
 function topage(p)
   local last = p.mode
   if last == page then return end
@@ -108,7 +129,7 @@ function topage(p)
   elseif last == cm_pending then
     local pending = p.pending_matrix
     if pending[1] ~= 1 or pending[2] ~= 0 or pending[3] ~= 0 or pending[4] ~= 1 or pending[5] ~= 0 or pending[6] ~= 0 then
-      p.strings[#p.strings+1] = format("%f %f %f %f %f %f cm", pending[1], pending[2], pending[3], pending[4], sp2bp(pending[5]), sp2bp(pending[6]))
+      p.strings[#p.strings+1] = strip_floats(format("%f %f %f %f %f %f cm", pending[1], pending[2], pending[3], pending[4], to_bp(pending[5]), to_bp(pending[6])))
     end
   else
     error[[Unknown mode]]
@@ -119,7 +140,7 @@ local function toglyph(p, fid, x, y, exfactor)
   local last = p.mode
   if last == glyph and p.font.fid == fid and p.pos.y == y and p.font.exfactor == exfactor then
     if x == p.pos.x then return end
-    local xoffset = (x - p.pos.x)/p.font.font.size * 1000 / (1+exfactor/1000000)
+    local xoffset = (x - p.pos.x)/p.font.font.size * 1000 / p.font.extend / (1+exfactor/1000000)
     if math.abs(xoffset) < 1000000 then -- 1000000 is arbitrary
       p.pending[#p.pending+1] = format(")%i(", math.floor(-xoffset))
       p.pos.x = x
@@ -128,14 +149,31 @@ local function toglyph(p, fid, x, y, exfactor)
   end
   if totext(p, fid) or exfactor ~= p.font.exfactor then
     p.font.exfactor = exfactor
-    p.strings[#p.strings+1] = gsub(format("%f 0.0 %f %f %f %f Tm", 1+exfactor/1000000, 0, 1, sp2bp(x), sp2bp(y)), '%.?0+ ', ' ')
+    p.strings[#p.strings+1] = strip_floats(format("%f 0.0 %f %f %f %f Tm", p.font.extend * (1+exfactor/1000000), p.font.slant, p.font.squeeze, to_bp(x), to_bp(y)))
   else
-    p.strings[#p.strings+1] = gsub(format("%f %f Td", sp2bp((x - p.pos.lx)/(1+exfactor/1000000)), sp2bp(y - p.pos.ly)), '%.?0+ ', ' ')
+    -- To invert the text transformation matrix (extend 0 0;slant squeeze 0;0 0 1)
+    -- we have to apply (extend^-1 0 0;-slant*extend^-1*squeeze^-1 squeeze^-1 0;0 0 1). (extend has to include expansion)
+    -- We optimize slightly by separating some steps
+    local dx, dy = to_bp((x - p.pos.lx)), to_bp(y - p.pos.ly) / p.font.squeeze
+    dx = (dx-p.font.slant*dy) / (p.font.extend * (1+exfactor/1000000))
+    p.strings[#p.strings+1] = strip_floats(format("%f %f Td", dx, dy))
   end
   p.pos.lx, p.pos.ly, p.pos.x, p.pos.y = x, y, x, y
   p.mode = glyph
   p.pending[1] = "[("
 end
+-- Let's start with "handlers" for nodes which do not need any special handling:
+local function ignore_node() end
+-- The following are already handled by the list handler because they only correspond to blank space:
+nodehandler.math = ignore_node
+nodehandler.kern = ignore_node
+-- The following are only for frontend use:
+nodehandler.boundary = ignore_node
+nodehandler.local_par = ignore_node
+nodehandler.penalty = ignore_node
+nodehandler.mark = ignore_node
+
+-- Now we come to more interesting nodes:
 function nodehandler.hlist(p, list, x0, y, outerlist, origin, level)
   if outerlist then
     if getid(outerlist) == 0 then
@@ -218,27 +256,43 @@ function nodehandler.vlist(p, list, x, y0, outerlist, origin, level)
     y = y - (d or 0)
   end
 end
+do
+local rulesubtypes = {}
+for i, n in next, node.subtypes'rule' do
+  rulesubtypes[n] = i
+end
+local box_rule = rulesubtypes.box
+local image_rule = rulesubtypes.image
+local user_rule = rulesubtypes.user
+local empty_rule = rulesubtypes.empty
+local outline_rule = rulesubtypes.outline
+local ship_img = require'luametalatex-pdf-image'.ship
+local ship_box = require'luametalatex-pdf-savedbox'.ship
+-- print(require'inspect'(node.subtypes('glue')))
+-- print(require'inspect'(node.fields('glue')))
+-- print(require'inspect'(node.fields('rule')))
+-- print(require'inspect'(node.fields('whatsit')))
 function nodehandler.rule(p, n, x, y, outer)
   if getwidth(n) == -1073741824 then setwidth(n, getwidth(outer)) end
   if getheight(n) == -1073741824 then setheight(n, getheight(outer)) end
   if getdepth(n) == -1073741824 then setdepth(n, getdepth(outer)) end
   local sub = getsubtype(n)
-  if sub == 1 then
-    error[[We can't handle boxes yet]]
-  elseif sub == 2 then
-    error[[We can't handle images yet]]
-  elseif sub == 3 then
-  elseif sub == 4 then
+  if getwidth(n) <= 0 or getdepth(n) + getheight(n) <= 0 then return end
+  if sub == box_rule then
+    ship_box(getdata(n), p, n, x, y)
+  elseif sub == image_rule then
+    ship_img(getdata(n), p, n, x, y)
+  elseif sub == empty_rule then
+  elseif sub == user_rule then
     error[[We can't handle user rules yet]]
-  elseif sub == 9 then
+  elseif sub == outline_rule then
     error[[We can't handle outline rules yet]]
   else
-    if getwidth(n) <= 0 or getdepth(n) + getheight(n) <= 0 then return end
     topage(p)
-    p.strings[#p.strings+1] = gsub(format("%f %f %f %f re f", sp2bp(x), sp2bp(y - getdepth(n)), sp2bp(getwidth(n)), sp2bp(getdepth(n) + getheight(n))), '%.?0+ ', ' ')
+    p.strings[#p.strings+1] = strip_floats(format("%f %f %f %f re f", to_bp(x), to_bp(y - getdepth(n)), to_bp(getwidth(n)), to_bp(getdepth(n) + getheight(n))))
   end
 end
-function nodehandler.boundary() end
+end
 function nodehandler.disc(p, n, x, y, list, ...) -- FIXME: I am not sure why this can happen, let's assume we can use .replace
   for n in traverse(getreplace(n)) do
     local next = getnext(n)
@@ -247,8 +301,6 @@ function nodehandler.disc(p, n, x, y, list, ...) -- FIXME: I am not sure why thi
     x = w + x
   end
 end
-function nodehandler.local_par() end
-function nodehandler.math() end
 function nodehandler.glue(p, n, x, y, outer, origin, level) -- Naturally this is an interesting one.
   local subtype = getsubtype(n)
   if subtype < 100 then return end -- We only really care about leaders
@@ -322,8 +374,6 @@ function nodehandler.glue(p, n, x, y, outer, origin, level) -- Naturally this is
     end
   end
 end
-function nodehandler.kern() end
-function nodehandler.penalty() end
 
 local pdf_escape = require'luametalatex-pdf-escape'.escape_raw
 local match = lpeg.match
@@ -333,31 +383,34 @@ local function do_commands(p, c, f, fid, x, y, outer, ...)
   for _, cmd in ipairs(c.commands) do
     if cmd[1] == "node" then
       local cmd = cmd[2]
+      assert(node.type(cmd))
+      cmd = todirect(cmd)
       nodehandler[getid(cmd)](p, cmd, x, y, nil, ...)
       x = x + getwidth(cmd)
     elseif cmd[1] == "font" then
-      current_font = fonts[cmd[2]]
+      current_font = assert(fonts[cmd[2]], "invalid font requested")
     elseif cmd[1] == "char" then
       local n = direct.new'glyph'
       setsubtype(n, 256)
       setfont(n, current_font.id, cmd[2])
       nodehandler.glyph(p, n, x, y, outer, ...)
-      direct.free(n)
       x = x + getwidth(n)
+      direct.free(n)
     elseif cmd[1] == "slot" then
+      current_font = assert(fonts[cmd[2]], "invalid font requested")
       local n = direct.new'glyph'
       setsubtype(n, 256)
-      setfont(n, cmd[2], cmd[3])
+      setfont(n, current_font.id, cmd[3])
       nodehandler.glyph(p, n, x, y, outer, ...)
-      direct.free(n)
       x = x + getwidth(n)
+      direct.free(n)
     elseif cmd[1] == "rule" then
       local n = direct.new'rule'
       setheight(n, cmd[2])
       setwidth(n, cmd[3])
       nodehandler.rule(p, n, x, y, outer, ...)
-      direct.free(n)
       x = x + getwidth(n)
+      direct.free(n)
     elseif cmd[1] == "left" then
       x = x + cmd[2]
     elseif cmd[1] == "down" then
@@ -383,10 +436,6 @@ local function do_commands(p, c, f, fid, x, y, outer, ...)
     -- else
       -- NOP, comment and invalid commands ignored
     end
-    if #commands ~= 1 then error[[Unsupported command number]] end
-    if commands[1][1] ~= "node" then error[[Unsupported command name]] end
-    commands = commands[1][2]
-    nodehandler[getid(commands)](p, commands, x, y, nil, ...)
   end
 end
 function nodehandler.glyph(p, n, x, y, ...)
@@ -417,21 +466,21 @@ function nodehandler.glyph(p, n, x, y, ...)
     else
       p.pending[#p.pending+1] = pdf_escape(string.pack('>H', index))
     end
-    if not p.usedglyphs[index] then
-      p.usedglyphs[index] = {index, math.floor(c.width * 1000 / f.size + .5), c.tounicode}
+    if not p.font.usedglyphs[index] then
+      p.font.usedglyphs[index] = {index, math.floor(c.width * 1000 / f.size / p.font.extend + .5), c.tounicode}
     end
   else
     p.pending[#p.pending+1] = pdf_escape(string.char(getchar(n)))
-    if not p.usedglyphs[getchar(n)] then
-      p.usedglyphs[getchar(n)] = {getchar(n), math.floor(c.width * 1000 / f.size + .5), c.tounicode}
+    if not p.font.usedglyphs[getchar(n)] then
+      p.font.usedglyphs[getchar(n)] = {getchar(n), math.floor(c.width * 1000 / f.size / p.font.extend + .5), c.tounicode}
     end
   end
   p.pos.x = p.pos.x + math.floor(getwidth(n)*(1+getexpansion(n)/1000000)+.5)
 end
 function nodehandler.whatsit(p, n, ...) -- Whatsit?
-  local prop = properties[n]-- or node.getproperty(n)
-  if prop and prop.handle then
-    prop:handle(p, n, ...)
+  local handler, prop = get_whatsit_handler(n)
+  if handler then
+    handler(prop, p, n, ...)
   else
     write("Invalid whatsit found (missing handler).")
   end
@@ -455,8 +504,12 @@ function pdf.write_matrix(a, b, c, d, e, f, p)
   pending[1], pending[2], pending[3], pending[4], pending[5], pending[6] = a, b, c, d, e, f
 end
 local write_matrix = pdf.write_matrix
+local literal_type_names = { [0] =
+  'origin', 'page', 'direct', 'raw', 'text'
+}
 function pdf.write(mode, text, x, y, p)
   x, y, p = x or global_x, y or global_y, p or global_p
+  mode = literal_type_names[mode] or mode
   if mode == "page" then
     topage(p)
     p.strings[#p.strings + 1] = text
@@ -483,24 +536,9 @@ local ondemandmeta = {
     return t[k]
   end
 }
-local function writeresources(p)
-  local resources = p.resources
-  local result = {"<<"}
-  for kind, t in pairs(resources) do if next(t) then
-    result[#result+1] = format("/%s<<", kind)
-    for name, value in pairs(t) do
-      result[#result+1] = format("/%s %i 0 R", name, value)
-      t[name] = nil
-    end
-    result[#result+1] = ">>"
-  end end
-  result[#result+1] = ">>"
-  return concat(result)
-end
-local fontnames = setmetatable({}, {__index = function(t, k) local res = format("F%i", k) t[k] = res return res end})
-return function(file, n, fontdirs, usedglyphs, colorstacks)
+local function nodewriter(file, n, fontdirs, usedglyphs, colorstacks, resources)
   n = todirect(n)
-  setmetatable(usedglyphs, ondemandmeta)
+  resources = resources or make_resources()
   local p = {
     is_page = not not colorstacks,
     file = file,
@@ -508,19 +546,15 @@ return function(file, n, fontdirs, usedglyphs, colorstacks)
     strings = {},
     pending = {},
     pos = {},
-    fontprovider = function(p, f, fid)
-      if not f.parent then f.parent = pdf.getfontname(fid) end
-      p.resources.Font[fontnames[f.parent]] = fontdirs[f.parent]
-      p.strings[#p.strings+1] = format("/F%i %f Tf 0 Tr", f.parent, sp2bp(f.size)) -- TODO: Setting the mode, expansion, etc.
-      p.usedglyphs = usedglyphs[f.parent]
-    end,
     font = {},
     vfont = {},
     matrix = {1, 0, 0, 1, 0, 0},
     pending_matrix = {},
-    resources = setmetatable({}, ondemandmeta),
+    resources = resources,
     annots = {},
     linkcontext = file.linkcontext,
+    fontdirs = fontdirs,
+    usedglyphs = usedglyphs,
   }
   if colorstacks then
     for i=1, #colorstacks do
@@ -536,5 +570,7 @@ return function(file, n, fontdirs, usedglyphs, colorstacks)
   nodehandler[getid(n)](p, n, 0, 0, n, nil, 0)
   -- nodehandler[getid(n)](p, n, 0, getdepth(n), n)
   topage(p)
-  return concat(p.strings, '\n'), writeresources(p), (p.annots[1] and string.format("/Annots[%s]", table.concat(p.annots, ' ')) or "")
+  return concat(p.strings, '\n'), resources, (p.annots[1] and string.format("/Annots[%s]", table.concat(p.annots, ' ')) or "")
 end
+require'luametalatex-pdf-savedbox':init_nodewriter(nodewriter)
+return nodewriter

luametalatex-pdf-escape.lua

@@ -1,4 +1,4 @@
-local mode = 6
+local mode = 0
 -- Control how much escaping is done... the mode is a bitset:
 -- Bit 0: Disable auto-detection of pre-escaped input
 -- Bit 1: Convert UTF-8 input to UTF-16

luametalatex-pdf-font-cff.lua

@@ -457,7 +457,7 @@ function myfunc(buf, i0, fontid, usedcids, encoding, trust_widths)
     if encoding == true then -- Use the built-in encoding
       CharStrings = parse_encoding(buf, i0, top.Encoding, CharStrings)
     elseif encoding then
-      encoding = require'parseEnc'(encoding)
+      encoding = require'luametalatex-font-enc'(encoding)
       local encoded = {}
       for i, n in pairs(encoding) do
         encoded[i] = CharStrings[n]
@@ -580,7 +580,7 @@ end
 -- local buf = file:read'a'
 -- file:close()
 -- io.open(arg[3], 'w'):write(myfunc(buf, 1, 1, nil, {{3}, {200}, {1000}, {1329}, {1330}, {1331}})):close()
-return function(filename, encoding) return function(fontdir, usedcids)
+return function(filename, fontid, encoding) return function(fontdir, usedcids)
   local file = io.open(filename)
   local buf = file:read'a'
   local i = 1
@@ -589,12 +589,12 @@ return function(filename, encoding) return function(fontdir, usedcids)
   if magic == "ttcf" or magic == "OTTO" then
     -- assert(not encoding) -- nil or false
     encoding = encoding or false
-    local magic, tables = sfnt.parse(buf, 1) -- TODO: Interpret widths etc, they might differ from the CFF ones.
+    local magic, tables = sfnt.parse(buf, fontid) -- TODO: Interpret widths etc, they might differ from the CFF ones.
     assert(magic == "OTTO")
     -- Also CFF2 would be nice to have
     i = tables['CFF '][1]
   end
-  local content, bbox = myfunc(buf, i, 1, usedcids, encoding)
+  local content, bbox = myfunc(buf, i, fontid, usedcids, encoding)
   fontdir.bbox = bbox
   return content
 end end

luametalatex-pdf-font-deduplicate.lua

@@ -0,0 +1,48 @@
+require'luametalatex-font-resolve' -- Ensure that font.fonts exists
+
+local keymap = {}
+
+-- There are multiple criteria for sharing backend fonts:
+--  * Obviously encodingbytes have to be the same.
+--  * The filename better is the same too.
+--  * Similarly the index must be the same.
+--  * Specifically in the PK font case *without* fixed DPI,
+--    The size must be the same too.
+--  * For fontmap based fonts, compare the name field instead,
+--    of the normal filename, especially to capture encoding differences.
+--    An alternative might be to only take the encoding into account.
+--    This is also required for other fonts which might not be backed by
+--    traditional files
+local function build_sharekey(fontdir)
+  local encodingbytes = assert(fontdir.encodingbytes)
+  local key = string.format("%i:%s:", fontdir.encodingbytes, fontdir.format)
+  if encodingbytes == 1 then
+    if fontdir.format:sub(1,5) == "type3" then
+      return string.format("%s%i:%s", key, fontdir.size, fontdir.name)
+    end
+    key = string.format("%s%s:", key, fontdir.encoding or '')
+  end
+  key = string.format("%s%i:%s", key, fontdir.subfont or 1, fontdir.filename)
+  return key
+end
+
+local fonts = font.fonts
+local fontmap = setmetatable({}, {
+  __index = function(t, fid)
+    local key = build_sharekey(assert(fonts[fid]))
+    local mapped = keymap[key]
+    local share_parent
+    if mapped then
+      share_parent = t[mapped]
+    else
+      share_parent, keymap[key], t[fid] = fid, fid, fid
+    end
+    return share_parent
+  end,
+})
+
+function pdf.getfontname(fid)
+  return fontmap[fid]
+end
+
+return fontmap

luametalatex-pdf-font-map.lua

@@ -42,7 +42,13 @@ local function mapfile(filename, operator)
   for line in file:lines() do mapline(line, operator) end
   file:close()
 end
+local function reset()
+  for k in next, fontmap do
+    fontmap[k] = nil
+  end
+end
 return {
+  reset = reset,
   mapline = mapline,
   mapfile = mapfile,
   fontmap = fontmap

luametalatex-pdf-font-pk.lua

@@ -0,0 +1,41 @@
+local pk_global_resolution, pk_resolution_is_fixed
+local pdfvariable = pdf.variable
+
+local read_pk = require'luametalatex-font-pk'
+local strip_floats = require'luametalatex-pdf-utils'.strip_floats
+return function(pdf, fontdir, usedcids)
+  if not pk_global_resolution then
+    pk_global_resolution = pdfvariable.pkresolution
+    if not pk_global_resolution or pk_global_resolution == 0 then
+      pk_global_resolution = kpse.var_value'pk_dpi' or 72
+    end
+    local mode = pdfvariable.pkmode
+    pk_resolution_is_fixed = pdfvariable.pkfixeddpi ~= 0
+    kpse.init_prog("LUATEX", pk_global_resolution, pkmode ~= '' and pkmode or nil, nil) -- ?
+  end
+  local pk = read_pk(kpse.find_file(fontdir.name, 'pk', pk_resolution_is_fixed and pk_global_resolution or (pk_global_resolution*fontdir.size/fontdir.designsize+.5)//1))
+  local designsize = pk.designsize/1044654.326 -- 1044654.326=2^20*72/72.27 -- designsize in bp
+  local hscale = 65536/pk.hppp / designsize -- 65291.158=2^16*72/72.27
+  local vscale = 65536/pk.vppp / designsize -- 65291.158=2^16*72/72.27
+  local bbox = {0, 0, 0, 0}
+  local matrix = {hscale, 0, 0, vscale, 0, 0}
+  local widths = {}
+  local first_cid = usedcids[1][1]-1
+  local charprocs = {}
+  local prev = 0
+  for i=1,#usedcids do
+    local used = usedcids[i]
+    local glyph = pk[used[1]]
+    for j=prev+1,used[1]-first_cid-1 do
+      widths[j] = 0
+    end
+    prev = used[1]-first_cid
+    widths[prev] = glyph.dx/2^16
+    local lower, left, upper, right = glyph.voff - glyph.h, -glyph.hoff, glyph.voff, -glyph.hoff + glyph.w
+    bbox[1], bbox[2], bbox[3], bbox[4] = math.min(bbox[1], left), math.min(bbox[2], lower), math.max(bbox[3], right), math.max(bbox[4], upper)
+    charprocs[i] = string.format("/G%i %i 0 R", used[1], pdf:stream(nil, "", string.format("%i %i %i %i %i %i d1 %i 0 0 %i %i %i cm BI /W %i/H %i/IM true/BPC 1/D[1 0] ID %s EI",
+        glyph.dx/2^16, glyph.dy, left, lower, right, upper, glyph.w, glyph.h, left, lower, glyph.w, glyph.h, glyph.data
+    )))
+  end
+  return bbox, matrix, pdf:indirect(nil, strip_floats('[' .. table.concat(widths, ' ') .. ']')), '<<' .. table.concat(charprocs) .. '>>'
+end

luametalatex-pdf-font-t1.lua

@@ -42,7 +42,7 @@ return function(filename, reencode)
       reencode = kpse.find_file("8a.enc", "enc files")
     end
     if reencode then
-      parsed_t1.Encoding = require'parseEnc'(reencode)
+      parsed_t1.Encoding = require'luametalatex-font-enc'(reencode)
     end
     -- parsed_t1.Encoding[0] = ".notdef"
     local glyphs = {}

luametalatex-pdf-font-ttf.lua

@@ -83,7 +83,7 @@ return function(filename, fontid, reencode)
 
   return function(fontdir, usedcids)
     if reencode and string.unpack(">I4", buf, tables.post[1]) == 0x00020000 then
-      usedcids = readpostnames(buf, tables.post[1] + 32, usedcids, require'parseEnc'(reencode))
+      usedcids = readpostnames(buf, tables.post[1] + 32, usedcids, require'luametalatex-font-enc'(reencode))
     else
       usedcids = table.move(usedcids, 1, #usedcids, 1, {})
     end

luametalatex-pdf-font.lua

@@ -1,5 +1,5 @@
-local mapping = require'luametalatex-pdf-font-map'
-mapping.mapfile(kpse.find_file('pdftex.map', 'map', true))
+local strip_floats = require'luametalatex-pdf-utils'.strip_floats
+
 local tounicode = {
   [-3] = require'luametalatex-pdf-font-cmap3',
          require'luametalatex-pdf-font-cmap1',
@@ -60,6 +60,31 @@ local function fontdescriptor(pdf, basefont, fontdir, stream, kind)
     fontdir.StemV or 100, -- FIXME: How to determine StemV?
     kind, stream)
 end
+local function encodingtype3(pdf)
+  if not pdf.encodingtype3 then
+    pdf.encodingtype3 = string.format(" %i 0 R", pdf:indirect(nil, "\z
+      <</Differences[\z
+        0/G0/G1/G2/G3/G4/G5/G6/G7/G8/G9/G10/G11/G12/G13/G14/G15/G16\z
+        /G17/G18/G19/G20/G21/G22/G23/G24/G25/G26/G27/G28/G29/G30/G31\z
+        /G32/G33/G34/G35/G36/G37/G38/G39/G40/G41/G42/G43/G44/G45/G46\z
+        /G47/G48/G49/G50/G51/G52/G53/G54/G55/G56/G57/G58/G59/G60/G61\z
+        /G62/G63/G64/G65/G66/G67/G68/G69/G70/G71/G72/G73/G74/G75/G76\z
+        /G77/G78/G79/G80/G81/G82/G83/G84/G85/G86/G87/G88/G89/G90/G91\z
+        /G92/G93/G94/G95/G96/G97/G98/G99/G100/G101/G102/G103/G104/G105/G106\z
+        /G107/G108/G109/G110/G111/G112/G113/G114/G115/G116/G117/G118/G119/G120/G121\z
+        /G122/G123/G124/G125/G126/G127/G128/G129/G130/G131/G132/G133/G134/G135/G136\z
+        /G137/G138/G139/G140/G141/G142/G143/G144/G145/G146/G147/G148/G149/G150/G151\z
+        /G152/G153/G154/G155/G156/G157/G158/G159/G160/G161/G162/G163/G164/G165/G166\z
+        /G167/G168/G169/G170/G171/G172/G173/G174/G175/G176/G177/G178/G179/G180/G181\z
+        /G182/G183/G184/G185/G186/G187/G188/G189/G190/G191/G192/G193/G194/G195/G196\z
+        /G197/G198/G199/G200/G201/G202/G203/G204/G205/G206/G207/G208/G209/G210/G211\z
+        /G212/G213/G214/G215/G216/G217/G218/G219/G220/G221/G222/G223/G224/G225/G226\z
+        /G227/G228/G229/G230/G231/G232/G233/G234/G235/G236/G237/G238/G239/G240/G241\z
+        /G242/G243/G244/G245/G246/G247/G248/G249/G250/G251/G252/G253/G254/G255]>>"
+    ))
+  end
+  return pdf.encodingtype3
+end
 local function cidmap1byte(pdf)
   if not pdf.cidmap1byte then
     pdf.cidmap1byte = string.format(" %i 0 R", pdf:stream(nil, [[/Type/CMap/CMapName/Identity-8-H/CIDSystemInfo<</Registry(Adobe)/Ordering(Identity)/Supplement 0>>]],
@@ -158,7 +183,7 @@ local function buildfont0cff(pdf, fontdir, usedcids)
       if fontdir.format == "type1" then
         cff = require'luametalatex-pdf-font-t1'(fontdir.filename, fontdir.encoding)(fontdir, usedcids)
       elseif fontdir.format == "opentype" then
-        cff = require'luametalatex-pdf-font-cff'(fontdir.filename, fontdir.encodingbytes == 1 and (fontdir.encoding or true))(fontdir, usedcids)
+        cff = require'luametalatex-pdf-font-cff'(fontdir.filename, 1, fontdir.encodingbytes == 1 and (fontdir.encoding or true))(fontdir, usedcids)
       else
         error[[Unsupported format]]
       end
@@ -224,50 +249,36 @@ local function buildfont0(pdf, fontdir, usedcids)
     touni,
     cidfont)
 end
-local fontextensions = {
-  ttf = {"truetype", "truetype fonts",},
-  otf = {"opentype", "opentype fonts",},
-  pfb = {"type1", "type1 fonts",},
-}
-fontextensions.cff = fontextensions.otf
-local fontformats = {
-  fontextensions.pfb, fontextensions.otf, fontextensions.ttf,
-}
+local buildfontpk = require'luametalatex-pdf-font-pk'
+local buildfontnode = require'luametalatex-pdf-font-node'.buildfont
+local function buildfont3(pdf, fontdir, usedcids)
+  local buildfont = fontdir.format == 'type3' and buildfontpk
+                 or fontdir.format == 'type3node' and buildfontnode
+                 or error[[Unsupported Type3 based font format]]
+  usedcids = usedcids or allcids(fontdir)
+  table.sort(usedcids, function(a,b) return a[1]<b[1] end)
+  local enc = cidmap1byte(pdf)
+  local bbox, matrix, widths, charprocs = buildfont(pdf, fontdir, usedcids)
+  local touni = pdf:stream(nil, "", tounicode[1](fontdir, usedcids)) -- Done late to allow for defaults set from the font file
+  local matrices = strip_floats(string.format(
+    "/FontBBox[%f %f %f %f]/FontMatrix[%f %f %f %f %f %f]",
+    bbox[1], bbox[2], bbox[3], bbox[4],
+    matrix[1], matrix[2], matrix[3], matrix[4], matrix[5], matrix[6]))
+  return string.format(
+    "<</Type/Font/Subtype/Type3%s/CharProcs%s/Encoding%s/FirstChar %i/LastChar %i/Widths %i 0 R/ToUnicode %i 0 R>>",
+    -- "<</Type/Font/Subtype/Type3/CharProcs%s/Encoding%s/FirstChar %i/LastChar %i/Widths %i 0 R/ToUnicode %i 0 R/FontDescriptor %i 0 R>>",
+    matrices,
+    charprocs,
+    encodingtype3(pdf),
+    usedcids[1][1],
+    usedcids[#usedcids][1],
+    widths,
+    touni
+    ) -- , descriptor) -- TODO
+end
 return function(pdf, fontdir, usedcids)
-  if fontdir.encodingbytes == 0 then fontdir.encodingbytes = nil end
-  if fontdir.format == "unknown" or not fontdir.format or fontdir.encodingbytes == 1 then -- TODO: How to check this?
-    fontdir.encodingbytes = fontdir.encodingbytes or 1
-    local mapentry = mapping.fontmap[fontdir.name]
-    if mapentry then
-      local format = mapentry[3] and mapentry[3]:sub(-4, -4) == '.' and fontextensions[mapentry[3]:sub(-3, -1)]
-      if format then
-        fontdir.format = format[1]
-        fontdir.filename = kpse.find_file(mapentry[3], format[2])
-        if mapentry[4] then
-          fontdir.encoding = kpse.find_file(mapentry[4], 'enc files')
-        end
-        goto format_set
-      else
-        for _, format in ipairs(fontformats) do
-          local font = kpse.find_file(mapentry[3],format[2])
-          if font then
-            fontdir.format = "type1"
-            fontdir.filename = font
-            if mapentry[4] then
-              fontdir.encoding = kpse.find_file(mapentry[4], 'enc files')
-            end
-            goto format_set
-          end
-        end
-      end
-    end
-    fontdir.format = "type3"
-    ::format_set::
-  else
-    fontdir.encodingbytes = fontdir.encodingbytes or 2
-  end
-  if fontdir.format == "type3" then
-    error[[Currently unsupported]] -- TODO
+  if fontdir.format:sub(1,5) == "type3" then
+    return buildfont3(pdf, fontdir, usedcids)
   else
     return buildfont0(pdf, fontdir, usedcids)
   end

luametalatex-pdf-image-pdf.lua

@@ -0,0 +1,108 @@
+local box_fallback = {
+  BleedBox = "CropBox",
+  TrimBox  = "CropBox",
+  ArtBox   = "CropBox",
+  CropBox  = "MediaBox",
+}
+  
+local boxmap = {
+  media = "MediaBox",
+  crop  = "CropBox",
+  bleed = "BleedBox",
+  trim  = "TrimBox",
+  art   = "ArtBox",
+}
+
+local utils = require'luametalatex-pdf-utils'
+local strip_floats = utils.strip_floats
+local to_sp = utils.to_sp
+local to_bp = utils.to_bp
+
+local function get_box(page, box)
+  box = boxmap[box]
+  while box do
+    local found = pdfe.getbox(page, box)
+    if found then
+      return {to_sp(found[1]), to_sp(found[2]), to_sp(found[3]), to_sp(found[4])}
+    end
+    box = box_fallback[box]
+  end
+end
+
+local pdf_functions = {}
+
+local function open_pdfe(img)
+  local file = pdfe.open(img.filepath)
+  do
+    local userpassword = img.userpassword
+    local ownerpassword = img.ownerpassword
+    if userpassword or ownerpassword then
+      pdfe.unencrypt(file, userpassword, ownerpassword)
+    end
+  end
+  local status = pdfe.getstatus(file)
+  if status >= 0 then
+    return file
+  elseif status == -1 then
+    error[[PDF image is encrypted. Please provide the decryption key.]]
+  elseif status == -2 then
+    error[[PDF image could not be opened.]]
+  else
+    assert(false)
+  end
+end
+function pdf_functions.scan(img)
+  local file = open_pdfe(img)
+  img.pages = pdfe.getnofpages(file)
+  img.page = img.page or 1
+  if img.page > img.pages then
+    error[[Not enough pages in PDF image]]
+  end
+  local page = pdfe.getpage(file, img.page)
+  local bbox = img.bbox or get_box(page, img.pagebox or 'crop') or {0, 0, 0, 0}
+  img.bbox = bbox
+  img.rotation = (360 - (page.Rotate or 0)) % 360
+  assert(img.rotation % 90 == 0, "Invalid /Rotate")
+  img.rotation = img.rotation / 90
+  if img.rotation < 0 then img.rotation = img.rotation + 4 end
+  img.xsize = bbox[3] - bbox[1]
+  img.ysize = bbox[4] - bbox[2]
+  img.xres, img.yres = nil, nil
+end
+
+local pdfe_deepcopy = require'luametalatex-pdfe-deepcopy'
+function pdf_functions.write(pfile, img)
+  local file = open_pdfe(img)
+  local page = pdfe.getpage(file, img.page)
+  local bbox = img.bbox
+  local dict = strip_floats(string.format("/Subtype/Form/BBox[%f %f %f %f]/Resources ", to_bp(bbox[1]), to_bp(bbox[2]), to_bp(bbox[3]), to_bp(bbox[4])))
+  dict = dict .. pdfe_deepcopy(file, img.filepath, pfile, pdfe.getfromdictionary(page, 'Resources'))
+  local content, raw = page.Contents
+  -- Three cases: Contents is a stream, so copy the stream (Remember to copy filter if necessary)
+  --              Contents is an array of streams, so append all the streams as a new stream
+  --              Contents is missing. Then create an empty stream.
+  local type = pdfe.type(content)
+  if type == 'pdfe.stream' then
+    raw = true
+    for i=1,#content do
+      local key, type, value, detail = pdfe.getfromstream(content, i)
+      dict = dict .. pdfe_deepcopy(file, img.filepath, pfile, 5, key) .. ' ' .. pdfe_deepcopy(file, img.filepath, pfile, type, value, detail)
+    end
+    content = content(false)
+  elseif type == 'pdfe.array' then
+    local array = content
+    content = ''
+    for i=1,#array do
+      content = content .. array[i](true)
+    end
+  else
+    content = ''
+  end
+  local attr = img.attr
+  if attr then
+    dict = dict .. attr
+  end
+  pfile:stream(img.objnum, dict, content, nil, raw)
+end
+
+return pdf_functions

luametalatex-pdf-image-png.lua

@@ -0,0 +1,351 @@
+local strip_floats = require'luametalatex-pdf-utils'.strip_floats
+
+local function ignore() end
+local parse = setmetatable({
+  -- IHDR = below,
+  -- PLTE = below,
+  -- IDAT = below,
+  -- IEND = below,
+  -- I'm not yet sure what to do about the following four color management chunks:
+  -- These two will probably be ignored (if you care about this stuff, you probably
+  -- prefer an ICC profile anyway. Also especially cHRM requires some weird computations.)
+  -- cHRM = TODO, -- ignore?
+  -- gAMA = TODO, -- ignore?
+  -- iCCP is implemented, but profiles are not cached, so it might include the
+  -- same profile many times
+  -- iCCP = below,
+  -- I would expect sRGB to be the most common, but it is a bit complicated because
+  -- PDF seems to require us to ship an actual ICC profile to support sRGB. Maybe later.
+  -- sRGB = TODO,
+  sBIT = ignore,
+  bKGD = ignore, -- Background color. Ignored since we support transparency
+  hIST = ignore, -- Color histogram
+  -- tRNS = below,
+  -- pHYs = below, -- resolution information
+  sPLT = ignore, -- Suggested palette but we support full truetype
+  tIME = ignore, -- The following only store metadata
+  iTXt = ignore,
+  tEXt = ignore,
+  zTXt = ignore,
+}, {
+    __index = function(_, n)
+      print("Table " .. n .. " unsupported") -- FIXME: Handle extensions by detecing if they are critical etc.
+      return ignore
+    end
+  })
+function parse.IHDR(buf, i, after, ctxt)
+  if next(ctxt) then
+    error[[The header should come first]]
+  end
+  local compression, filter
+  ctxt.width, ctxt.height,
+  ctxt.bitdepth, ctxt.colortype,
+  compression, filter,
+  ctxt.interlace, i = string.unpack(">I4I4I1I1I1I1I1", buf, i)
+  if i ~= after then
+    return [[Invalid header size]]
+  end
+  if compression ~= 0 then
+    error [[Unsupported compression mode]]
+  end
+  if filter ~= 0 then
+    error [[Unsupported filter mode]]
+  end
+end
+function parse.PLTE(buf, i, after, ctxt)
+  if ctxt.PLTE then
+    error[[Multiple palettes detected]]
+  end
+  if (after-i)%3 ~= 0 then
+    error[[Invalid palette lenght]]
+  end
+  ctxt.PLTE_len = (after-i) // 3
+  ctxt.PLTE = string.sub(buf, i, after-1)
+end
+function parse.tRNS(buf, i, after, ctxt)
+  if ctxt.colortype == 3 then
+    local count = assert(ctxt.PLTE_len)
+    local patt = lpeg.P(1) * lpeg.Cc'\xff'
+    for j=0,after-i-1 do
+      local off = i+j
+      patt = lpeg.P(string.char(j)) * lpeg.Cc(buf:sub(off, off)) + patt
+    end
+    ctxt.tRNS = lpeg.Cs(lpeg.Cg(patt)^0)
+  elseif ctxt.colortype == 0 then
+    local color
+    color, i = string.unpack(">I2", buf, i)
+    assert(i == after)
+    ctxt.tRNS = string.format('%i %i', color, color)
+  elseif ctxt.colortype == 2 then
+    local r, g, b
+    r, g, b, i = string.unpack(">I2I2I2", buf, i)
+    assert(i == after)
+    ctxt.tRNS = string.format('%i %i %i %i %i %i', r, r, g, g, b, b)
+  end
+end
+local meterperinch = 0.0254
+function parse.pHYs(buf, i, after, ctxt)
+  local xres, yres, unit
+  xres, yres, unit, i = string.unpack('>I4I4I1', buf, i)
+  if unit == 0 then
+    if xres > yres then
+      ctxt.xres, ctxt.yres = xres/yres, 0
+    elseif xres < yres then
+      ctxt.xres, ctxt.yres = 0, yres/xres
+    end
+  elseif unit == 1 then
+    ctxt.xres, ctxt.yres = xres * meterperinch, yres * meterperinch
+  else
+    error[[Invalid unit]]
+  end
+  assert(i == after)
+end
+function parse.sRGB(buf, i, after, ctxt)
+  assert(i+1 == after)
+  ctxt.sRGB = buf:char(i)
+end
+function parse.iCCP(buf, i, after, ctxt)
+  local j = buf:find('\0', i, true)
+  assert(j+1<after)
+  -- local name = buf:sub(i, j-1)
+  -- print('ICC Profile name: ' .. name)
+  assert(buf:byte(j+1) == 0) -- The only known compression mode
+  ctxt.iCCP = buf:sub(j+2, after-1)
+  -- ctxt.iCCP = xzip.decompress(buf:sub(j+2, after-1))
+end
+function parse.gAMA(buf, i, after, ctxt)
+  local gamma, i = string.unpack(">I4", buf, i)
+  assert(after == i)
+  ctxt.gAMA = 100000/gamma
+end
+function parse.cHRM(buf, i, after, ctxt)
+  local x_W, y_W, x_R, y_R, x_G, y_G, x_B, y_B, i = string.unpack(">I4I4I4I4I4I4I4I4", buf, i)
+  assert(after == i)
+  x_W, y_W, x_R, y_R, x_G, y_G, x_B, y_B = x_W/100000, y_W/100000, x_R/100000, y_R/100000,
+                                           x_G/100000, y_G/100000, x_B/100000, y_B/100000
+  local z = y_W*((x_G-x_B)*y_R-(x_R-x_B)*y_G+(x_R-x_G)*y_B)
+  z = 1/z
+  local Y_A = y_R*((x_G-x_B)*y_W-(x_W-x_B)*y_G+(x_W-x_G)*y_B) * z
+  local X_A, Z_A = Y_A*x_R/y_R, Y_A*((1-x_R)/y_R-1)
+  local Y_B = -y_G*((x_R-x_B)*y_W-(x_W-x_B)*y_R+(x_W-x_R)*y_B) * z
+  local X_B, Z_B = Y_B*x_G/y_G, Y_B*((1-x_G)/y_G-1)
+  local Y_C = y_B*((x_R-x_G)*y_W-(x_W-x_G)*y_R+(x_W-x_R)*y_G) * z
+  local X_C, Z_C = Y_C*x_B/y_B, Y_C*((1-x_B)/y_B-1)
+
+  local X_W, Y_W, Z_W = X_A+X_B+X_C, Y_A+Y_B+Y_C, Z_A+Z_B+Z_C
+  ctxt.cHRM = strip_floats(string.format("/WhitePoint[%f %f %f]/Matrix[%f %f %f %f %f %f %f %f %f]",
+      X_W, Y_W, Z_W,
+      X_A, Y_A, Z_A,
+      X_B, Y_B, Z_B,
+      X_C, Y_C, Z_C))
+end
+function parse.IDAT(buf, i, after, ctxt)
+  ctxt.IDAT = ctxt.IDAT or {}
+  table.insert(ctxt.IDAT, buf:sub(i, after-1))
+end
+function parse.IEND(buf, i, after)
+  if i ~= after then
+    error[[Unexpected data in end chunk]]
+  end
+end
+
+local function run(buf, i, len, limit)
+  i = i or 1
+  len = i+(len or #buf)
+  if buf:sub(i,i+7) ~= "\x89PNG\x0D\x0A\x1A\x0A" then
+    error[[You lied. This isn't a PNG file.]]
+  end
+  i = i+8
+  local chunks = {}
+  while i < len do
+    local length, tp, off = string.unpack(">I4c4", buf, i)
+    if tp == limit then break end
+    parse[tp](buf, off, off + length, chunks)
+    i = off + length + 4
+  end
+  return chunks, i
+end
+local function passes(buf, width, height, bitdepth, colortype)
+  local stride = (bitdepth == 16 and 2 or 1) * (1 + (colortype&3 == 2 and 2 or 0) + (colortype&4)/4)
+  local passes = {
+    {(width+7)//8, (height+7)//8},
+    {(width+3)//8, (height+7)//8},
+    {(width+3)//4, (height+3)//8},
+    {(width+1)//4, (height+3)//4},
+    {(width+1)//2, (height+1)//4},
+    { width   //2, (height+1)//2},
+    { width      ,  height   //2},
+  }
+  local off = 1
+  local result
+  for i=1,#passes do
+    local xsize, ysize = passes[i][1], passes[i][2]
+    if xsize ~= 0 and ysize ~= 0 then
+      if bitdepth < 8 then
+        xsize = (xsize * bitdepth + 7) // 8
+      end
+      local after = off + (xsize+1) * stride * ysize
+      local pass = pngdecode.applyfilter(
+        buf:sub(off, after-1),
+        xsize,
+        ysize,
+        stride)
+      if bitdepth < 8 then
+        pass = pngdecode.expand(pass, passes[i][1], ysize, bitdepth, xsize)
+      end
+      result = pngdecode.interlace(width, height, stride, i, pass, result)
+      off = after
+    end
+  end
+  assert(off == #buf+1)
+  return result
+end
+
+local png_functions = {}
+
+function png_functions.scan(img)
+  local file = io.open(img.filepath)
+  if not file then
+    error[[PDF image could not be opened.]]
+  end
+  local buf = file:read'a'
+  file:close()
+  local t = run(buf, 1, #buf, 'IDAT')
+  img.pages = 1
+  img.page = 1
+  img.rotation = 0
+  img.xsize, img.ysize = t.width, t.height
+  img.xres, img.yres = t.xres or 0, t.yres or 0
+  img.colordepth = t.bitdepth
+end
+
+local srgb_colorspace
+local intents = {[0]=
+  '/Intent/Perceptual',
+  '/Intent/RelativeColorimetric',
+  '/Intent/Saturation',
+  '/Intent/AbsoluteColorimetric',
+}
+local function srgb_lookup(pfile, intent)
+  if not srgb_colorspace then
+    local f = io.open(kpse.find_file'sRGB.icc.zlib')
+    local profile = f:read'a'
+    f:close()
+    local objnum = pfile:stream(nil, '/Filter/FlateDecode/N ' .. tostring(colortype & 2 == 2 and '3' or '1'), t.iCCP, nil, true)
+    srgb_colorspace = string.format('[/ICCBased %i 0 R]', objnum)
+  end
+  return objnum, intents[intent] or ''
+end
+
+local pdf_escape = require'luametalatex-pdf-escape'.escape_bytes
+
+local function rawimage(t, content)
+  content = xzip.decompress(content)
+  if t.interlace == 1 then
+    content = passes(content, t.width, t.height, t.bitdepth, t.colortype)
+  else
+    local xsize = t.width
+    if t.bitdepth < 8 then
+      xsize = (xsize * t.bitdepth + 7) // 8
+    end
+    local colortype = t.colortype
+    content = pngdecode.applyfilter(
+      content,
+      xsize,
+      t.height,
+      (t.bitdepth == 16 and 2 or 1) * (1 + (colortype&3 == 2 and 2 or 0) + (colortype&4)/4))
+  end
+  return content
+end
+
+function png_functions.write(pfile, img)
+  local file = io.open(img.filepath)
+  if not file then
+    error[[PDF image could not be opened.]]
+  end
+  local buf = file:read'a'
+  file:close()
+  local t = run(buf, 1, #buf, 'IEND')
+  local colorspace
+  local intent = ''
+  local colortype = t.colortype
+  if img.colorspace then
+    colorspace = string.format(' %i 0 R', img.colorspace)
+  elseif t.iCCP then
+    local icc_ref = pfile:stream(nil, '/Filter/FlateDecode/N ' .. tostring(colortype & 2 == 2 and '3' or '1'), t.iCCP, nil, true)
+    colorspace = string.format('[/ICCBased %i 0 R]', icc_ref)
+  elseif t.sRGB then
+    colorspace, intent = srgb_lookup(pfile, t.sRGB)
+  elseif colortype & 2 == 2 then -- RGB
+    if t.cHRM then
+      local gamma = t.gAMA or 2.2
+      gamma = gamma and strip_floats(string.format("/Gamma[%f %f %f]", gamma, gamma, gamma)) or ''
+      colorspace = string.format("[/CalRGB<<%s%s>>]", t.cHRM, gamma)
+    else
+      if t.gAMA then
+        texio.write_nl'Warning: (PNG) Gamma correction without chromaticity information is unsupported. Gamma value will be ignored.'
+      end
+      colorspace = '/DeviceRGB'
+    end
+  else -- Gray
+    if t.gAMA or t.cHRM then
+      texio.write_nl'Warning: (PNG) Gamma correction and chromaticity specifications are only supported for RGB images.'
+    end
+    colorspace = '/DeviceGray'
+  end
+  if colortype & 1 == 1 then -- Indexed
+    colorspace = string.format('[/Indexed%s %i%s]', colorspace, t.PLTE_len-1, pdf_escape(t.PLTE))
+  end
+  local colordepth = t.interlace == 1 and 8 or img.colordepth
+  local dict = string.format("/Subtype/Image/Width %i/Height %i/BitsPerComponent %i/ColorSpace%s", img.xsize, img.ysize, colordepth, colorspace)
+
+  local content = table.concat(t.IDAT)
+  local copy -- = true
+  if copy and (t.interlace == 1 or colortype & 4 == 4) then -- TODO: Add additional conditions
+    copy = false
+  end
+
+  if copy then
+    -- In this case we never have to deal with an alpha component
+    dict = string.format(
+        '%s/Filter/FlateDecode/DecodeParms<</Colors %i/Columns %i/BitsPerComponent %i/Predictor 10>>',
+        dict, colortype == 2 and 3 or 1, img.xsize, colordepth)
+  else
+    content = rawimage(t, content)
+    if colortype & 4 == 4 then -- Alpha channel present
+      local mask
+      content, mask = pngdecode.splitmask(
+        content,
+        img.xsize,
+        img.ysize,
+        1 + (colortype&2),
+        colordepth//8) -- colordepth must be 8 or 16 if alpha is present
+      local mask_dict = string.format("/Subtype/Image/Width %i/Height %i/BitsPerComponent %i/ColorSpace/DeviceGray", img.xsize, img.ysize, colordepth)
+      local objnum = pfile:stream(nil, mask_dict, mask)
+      dict = string.format('%s/SMask %i 0 R', dict, objnum)
+    end
+  end
+
+  if t.tRNS then
+    if colortype == 3 then
+      local unpacked = copy and rawimage(t, content) or content
+      if colordepth ~= 8 then
+        unpacked = pngdecode.expand(unpacked, img.xsize, img.ysize, colordepth, (img.xsize*colordepth+7)//8)
+      end
+      unpacked = t.tRNS:match(unpacked)
+      local mask_dict = string.format("/Subtype/Image/Width %i/Height %i/BitsPerComponent 8/ColorSpace/DeviceGray", img.xsize, img.ysize)
+      local objnum = pfile:stream(nil, mask_dict, unpacked)
+      dict = string.format('%s/SMask %i 0 R', dict, objnum)
+    else
+      dict = string.format('%s/Mask[%s]', dict, t.tRNS)
+    end
+  end
+
+  local attr = img.attr
+  if attr then
+    dict = dict .. attr
+  end
+  pfile:stream(img.objnum, dict, content, nil, copy)
+end
+
+return png_functions

luametalatex-pdf-image.lua

@@ -0,0 +1,217 @@
+local rawset = rawset
+local setdata = node.direct.setdata
+local nodenew = node.direct.new
+local getwhd = node.direct.getwhd
+local setwhd = node.direct.setwhd
+local tonode = node.direct.tonode
+local nodewrite = node.write
+
+-- Mapping extensions to canonical type names if necessary
+local imagetype_map = {
+  -- pdf1 = 'pdf',
+}
+local imagetypes = setmetatable({}, {__index = function(t, k)
+  local remapped = imagetype_map[k]
+  local module = remapped and t[remapped] or require('luametalatex-pdf-image-' .. k)
+  t[k] = module
+  return module
+end})
+
+local utils = require'luametalatex-pdf-utils'
+local strip_floats = utils.strip_floats
+local to_bp = utils.to_bp
+
+local liberal_keys = {height = true, width = true, depth = true, transform = true}
+local real_images = {}
+local function relaxed_newindex(t, k, v)
+  if liberal_keys[k] then
+    return rawset(t, k, v)
+  else
+    real_images[t][k] = v
+  end
+end
+local function no_newindex(t, k, v)
+  if liberal_keys[k] then
+    return rawset(t, k, v)
+  else
+    error(string.format("You are not allowed to set %q in an already scanned image"))
+  end
+end
+local function get_underlying(t, k)
+  return assert(real_images[t])[k]
+end
+local meta = {__index = get_underlying, __newindex = relaxed_newindex}
+local restricted_meta = {__index = get_underlying, __newindex = no_newindex}
+-- transform isn't documented to be changable but it kind of fits
+local function new(spec)
+  local img, real = {}, {}
+  real_images[img] = real
+  if spec then for k,v in next, spec do
+    (liberal_keys[k] and img or real)[k] = v
+  end end
+  img.depth = img.depth or 0
+  return setmetatable(img, meta)
+end
+local function scan(img)
+  local m = getmetatable(img)
+  local real
+  if m == restricted_meta then
+    real = real_images[img]
+  else
+    if m ~= meta then img = new(img) end
+    real = real_images[img]
+    if real.stream then error[[stream images are not yet supported]] end
+    assert(real.filename)
+    -- TODO: At some point we should just take the lowercased extension
+    local imagetype = real.filename:match'%.pdf$' and 'pdf'
+                   or real.filename:match'%.png$' and 'png'
+                   or error'Unsupported image format'
+    real.filepath = assert(kpse.find_file(real.filename), "Image not found")
+    real.imagetype = imagetype
+    imagetypes[imagetype].scan(real)
+    setmetatable(img, restricted_meta)
+  end
+  img.transform = img.transform or 0
+  -- (Re)Set dimensions
+  if img.depth and img.height and img.width then
+    return img
+  end
+  local flipped = (img.transform + real.rotation) % 2 == 1
+  if not (img.depth or img.height) then img.depth = 0 end
+  if not img.width and not (img.height and img.depth) then
+    local xsize, ysize = real.xsize, real.ysize
+    if not real.bbox then
+      local xres, yres = img.xres, img.yres
+      -- TODO: \pdfvariable Parameters
+      if xres == 0 then
+        xres = 72
+        yres = xres * ((not yres or yres == 0) and 1 or yres)
+      elseif yres == 0 then
+        yres = 72
+        xres = yres * ((not xres or xres == 0) and 1 or xres)
+      end
+      local xscale, yscale = 4736286.72/xres, 4736286.72/yres
+      xsize, ysize = xsize*xscale//1, ysize*yscale//1
+    end
+    local total_y
+    if flipped then
+      img.width = ysize
+      total_y = xsize
+    else
+      img.width = xsize
+      total_y = ysize
+    end
+    if img.height then
+      img.depth = total_y - img.height
+    else
+      img.height = total_y - img.depth
+    end
+  else
+    local ratio = flipped and xsize / ysize or ysize / xsize
+    if img.width then
+      if img.depth then
+        img.height = (ratio * img.width - img.depth) // 1
+      else
+        img.depth = (ratio * img.width - img.height) // 1
+      end
+    else
+      img.width = ((img.height + img.depth) / ratio) // 1
+    end
+  end
+  return img
+end
+
+local img_by_objnum = {}
+-- local function img_from_objnum(objnum, img)
+  -- img = img or {}
+  -- real_images[img] = assert(img_by_objnum[objnum])
+  -- return setmetatable(img, restricted_meta)
+-- end
+
+-- Noop if already reserved
+function reserve(pfile, real)
+  local obj = real.objnum or pfile:getobj()
+  real.objnum = obj
+  img_by_objnum[obj] = real
+  return obj
+end
+
+local function write_img(pfile, img)
+  local objnum = reserve(pfile, img)
+  if not img.written then
+    img.written = true
+    imagetypes[img.imagetype].write(pfile, img)
+  end
+end
+local function do_img(data, p, n, x, y)
+  local img = assert(img_by_objnum[data >> 3], 'Invalid image ID')
+  write_img(p.file, img)
+  local mirror = data & 4 == 4
+  local rotate = (data + img.rotation) & 3
+  local width, height, depth = getwhd(n)
+  height = height + depth
+  local bbox = img.bbox
+  local xsize, ysize = img.xsize, img.ysize
+  local a, b, c, d, e, f = 1, 0, 0, 1
+  if bbox then
+    e, f = -bbox[1], -bbox[2]
+  else
+    e, f = 0, 0
+    xsize, ysize = 65781.76, 65781.76
+  end
+  if mirror then
+    a, e = -a, -e+xsize
+  end
+  for i=1,rotate do
+    a, b, c, d, e, f = -b, a, -d, c, -f+ysize, e
+    xsize, ysize = ysize, xsize
+  end
+  local xscale, yscale = width / xsize, height / ysize
+  a, c, e = a*xscale, c*xscale, e*xscale
+  b, d, f = b*yscale, d*yscale, f*yscale
+  e, f = to_bp(x + e), to_bp(y - depth + f)
+  p.resources.XObject['Im' .. tostring(img.objnum)] = img.objnum
+  pdf.write('page', strip_floats(string.format('q %f %f %f %f %f %f cm /Im%i Do Q', a, b, c, d, e, f, img.objnum)), nil, nil, p)
+end
+local ruleid = node.id'rule'
+local ruletypes = node.subtypes'rule'
+local imagerule
+for n, name in next, ruletypes do
+  if name == 'image' then imagerule = n break end
+end
+assert(imagerule)
+local function node(pfile, img)
+  img = scan(img)
+  local n = nodenew(ruleid, imagerule) -- image
+  setdata(n, (reserve(pfile, real_images[img]) << 3) | ((img.transform or 0) & 7))
+  setwhd(n, img.width or -0x40000000, img.height or -0x40000000, img.depth or -0x40000000)
+  return tonode(n)
+end
+
+local function write(pfile, img)
+  img = scan(img)
+  nodewrite(node(pfile, img))
+  return img
+end
+
+local function immediatewrite(pfile, img)
+  img = scan(img)
+  write_img(pfile, real_images[img])
+  return img
+end
+
+return {
+  new = new,
+  scan = scan,
+  write = write,
+  node = node,
+  from_num = function(i)
+    local img = {}
+    real_images[img] = assert(img_by_objnum[i])
+    return setmetatable(img, restricted_meta)
+  end,
+  get_num = function(pfile, img)
+    return reserve(pfile, real_images[img])
+  end,
+  ship = do_img,
+}

luametalatex-pdf-pagetree.lua

@@ -1,6 +1,7 @@
 local min = math.min
 local format = string.format
 local concat = table.concat
+local pdfvariable = pdf.variable
 local function write(pdf, tree, total, max)
   tree = tree or pdf.pages
   if #tree == 0 then 
@@ -22,25 +23,47 @@ local function write(pdf, tree, total, max)
       local parentid = pdf:getobj()
       newtree[-(i//6)] = parentid
       parent = format("/Parent %i 0 R", parentid)
+    elseif #tree <= 6 then
+      parent = pdfvariable.pagesattr
     end
     pdf:indirect(id, format('<</Type/Pages%s/Kids[%s 0 R]/Count %i>>', parent, concat(tree, ' 0 R ', 6*i+1, min(#tree, 6*i+6)), min(remaining, max)))
     remaining = remaining - max
   end
-  if #parent > 0 then
+  if newtree[0] then
     return write(pdf, newtree, total, max*6)
   end
   return newtree[1]
 end
 local function newpage(pdf)
-  local pageid = pdf:getobj()
-  local pagenumber = #pdf.pages
-  pdf.pages[pagenumber+1] = pageid
-  if 0 == pagenumber % 6 then
-    pdf.pages[-(pagenumber//6)] = pdf:getobj()
+  local pages = pdf.pages
+  local pagenumber = #pages+1
+  local pageid = pages.reserved and pages.reserved[pagenumber]
+  if pageid then
+    pages.reserved[pagenumber] = nil
+  else
+    pageid = pdf:getobj()
   end
-  return pageid, pdf.pages[-(pagenumber//6)]
+  pages[pagenumber] = pageid
+  if 1 == pagenumber % 6 then
+    pages[-((pagenumber-1)//6)] = pdf:getobj()
+  end
+  return pageid, pages[-((pagenumber-1)//6)]
+end
+local function reservepage(pdf, num)
+  local pages = pdf.pages
+  if pages[num] then return pages[num] end
+  local reserved = pages.reserved
+  if reserved then
+    if reserved[num] then return reserved[num] end
+  else
+    reserved = {}
+    pages.reserved = reserved
+  end
+  reserved[num] = pdf:getobj()
+  return reserved[num]
 end
 return {
   write = write,
   newpage = newpage,
+  reservepage = reservepage,
 }

luametalatex-pdf-savedbox.lua

@@ -0,0 +1,106 @@
+local writer -- = require'luametalatex-nodewriter' -- This would introduce some cyclic dependency
+local pdfvariable = pdf.variable
+
+-- XForms currently have the form {width, height, depth, objnum, attributes, list, margin}
+local xforms = {}
+
+local utils = require'luametalatex-pdf-utils'
+local strip_floats = utils.strip_floats
+local to_bp = utils.to_bp
+
+local function shipout(pfile, xform, fontdirs, usedglyphs)
+  local list, margin = xform.list, xform.margin
+  if not list then return xform.objnum end -- Already shipped out
+  local last_page = cur_page cur_page = nil
+  local out, resources, annots = writer(pfile, list, fontdirs, usedglyphs)
+  cur_page = last_page
+  assert(annots == '')
+  if pdfvariable.xformattr ~= '' or pdfvariable.xformresources ~= '' then
+    texio.write_nl('term and log', 'WARNING (savedboxresource shipout): Ignoring unsupported PDF variables xformattr and xformresources. Specify resources and attributes for specific XForms instead.')
+  end
+  local bbox = strip_floats(string.format('/BBox[%f %f %f %f]', -to_bp(margin), -to_bp(list.depth+margin), to_bp(list.width+margin), to_bp(list.height+margin)))
+  local dict = string.format('/Subtype/Form%s/Resources%s%s', bbox, resources(xform.resources), xform.attributes or '')
+  node.flush_list(list)
+  xform.list = nil
+  local objnum = pfile:stream(xform.objnum, dict, out)
+  xform.objnum = objnum
+  return objnum
+end
+
+local function save(pfile, n, attr, resources, immediate, type, margin, fontdirs, usedglyphs)
+  local index = #xforms+1
+  local xform = {
+    list = assert(n, 'List required for saveboxresource'),
+    width = n.width,
+    height = n.height,
+    depth = n.depth,
+    attributes = attr,
+    resources = resources,
+    margin = margin,
+    -- type = type, -- TODO: Not yet used. Do we need this at all?
+  }
+  xforms[index] = xform
+  if immediate then
+    shipout(pfile, xform, fontdirs, usedglyphs)
+  end
+  return index
+end
+
+local function adjust_sizes(width, height, depth, real_width, real_height, real_depth)
+  if not depth then
+    if height then
+      local scale = height/real_height
+      depth = (real_depth*scale + .5)//1
+      width = width or (real_width*scale + .5)//1
+    elseif width then
+      local scale = width/real_width
+      depth = (real_depth*scale + .5)//1
+      height = (real_height*scale + .5)//1
+    else
+      width, height, depth = real_width, real_height, real_depth
+    end
+  elseif height then
+    width = width or (real_width*(height+depth)/(real_height+real_depth) + .5)//1
+  else
+    width = width or real_width
+    local scale = width/real_width
+    height = ((real_depth+real_height)*scale + .5)//1 - depth
+  end
+  return width, height, depth
+end
+
+local ruleid = node.id'rule'
+local ruletypes = node.subtypes'rule'
+local boxrule
+for n, name in next, ruletypes do
+  if name == 'box' then boxrule = n break end
+end   
+
+local function use(index, width, height, depth)
+  local xform = xforms[index]
+  if not xform then return nil, nil, nil, nil end
+  width, height, depth = adjust_sizes(width, height, depth, xform.width, xform.height, xform.depth)
+  local n = node.direct.new(ruleid, boxrule)
+  node.direct.setdata(n, index)
+  node.direct.setwhd(n, width, height, depth)
+  return node.direct.tonode(n), width, height, depth
+end
+
+local function do_box(data, p, n, x, y)
+  local xform = assert(xforms[data], 'Invalid XForm')
+  local objnum = shipout(p.file, xform, p.fontdirs, p.usedglyphs)
+  local width, height, depth = node.direct.getwhd(n)
+  local xscale, yscale = width / xform.width, (height+depth) / (xform.height+xform.depth)
+  p.resources.XObject['Fm' .. tostring(data)] = objnum
+  pdf.write('page', strip_floats(string.format('q %f 0 0 %f %f %f cm /Fm%i Do Q',
+    xscale, yscale,
+    to_bp(x), to_bp(y-depth+yscale*xform.depth),
+    data)), nil, nil, p)
+end
+
+return {
+  save = save,
+  use = use,
+  ship = do_box,
+  init_nodewriter = function(t, nodewriter) writer, t.init_nodewriter = nodewriter, nil end,
+}

luametalatex-pdf-utils.lua

@@ -0,0 +1,23 @@
+local l = lpeg or require'lpeg'
+local trailing_zeros = l.P'0'^0 * -l.R'09'
+local strip_floats_patt = l.Cs((1-l.R'09' +
+  (l.R'09')^1 * (l.P'.' * trailing_zeros / '' + l.P'.' * (l.R'09'-trailing_zeros)^1 * (trailing_zeros/''))^-1)^0)
+local match = l.match
+
+local function strip_floats(s)
+  return match(strip_floats_patt, s)
+end
+
+local function to_bp(sp)
+  return sp/65781.76
+end
+
+local function to_sp(bp)
+  return (bp*65781.76+.5)//1
+end
+
+return {
+  strip_floats = strip_floats,
+  to_bp = to_bp,
+  to_sp = to_sp,
+}

luametalatex-pdf.lua

@@ -1,5 +1,4 @@
 local format = string.format
-local gsub = string.gsub
 local byte = string.byte
 local pack = string.pack
 local error = error
@@ -7,6 +6,8 @@ local pairs = pairs
 local setmetatable = setmetatable
 local assigned = {}
 local delayed = {}
+local compress = xzip.compress
+local pdfvariable = pdf.variable
 -- slightly tricky interface: No/nil return means that the objects content
 -- isn't known yet, while false indicates a delayed object.
 local function written(pdf, num)
@@ -26,34 +27,57 @@ local function stream(pdf, num, dict, content, isfile, raw)
     content = f:read'a'
     f:close()
   end
-  pdf.file:write(format('%i 0 obj\n<<%s/Length %i>>stream\n', num, dict, #content))
+  local level = not raw and pdfvariable.compresslevel or 0
+  local filter = ''
+  if level > 0 then
+    local compressed = compress(content, level)
+    if #compressed < #content + 19 then -- Filter has some overhead
+      filter = "/Filter/FlateDecode"
+      content = compressed
+    end
+  end
+  pdf.file:write(format('%i 0 obj\n<<%s%s/Length %i>>stream\n', num, dict, filter, #content))
   pdf.file:write(content)
   pdf.file:write'\nendstream\nendobj\n'
   return num
 end
-local function delayedstream(pdf, num, dict, content, isfile)
+local function delayedstream(pdf, num, dict, content, isfile, raw)
   if not num then num = pdf:getobj() end
   if pdf[num] ~= assigned then
     error[[Invalid object]]
   end
   pdf[num] = delayed
-  pdf[-num] = {stream, dict, content, isfile}
+  pdf[-num] = {stream, dict, content, isfile, raw}
   return num
 end
-local function indirect(pdf, num, content, isfile)
+local function indirect(pdf, num, content, isfile, objstream)
   if not num then num = pdf:getobj() end
   if pdf[num] ~= assigned then
     error[[Invalid object]]
   end
-  pdf[num] = {offset = pdf.file:seek()}
-  pdf.file:write(format('%i 0 obj\n', num))
   if isfile then
     local f = io.open(content)
     content = f:read'a'
     f:close()
   end
-  pdf.file:write(content)
-  pdf.file:write'\nendobj\n'
+  if objstream ~= false and pdfvariable.objcompresslevel ~= 0 then
+    objstream = objstream or true
+    local objstr = pdf.objstream[objstream]
+    if not objstr then
+      objstr = {objnum = pdf:getobj(), off = 0, {}}
+      pdf.objstream[objstream] = objstr
+    end
+    local i = #objstr
+    pdf[num] = {objstr = objstr.objnum, i = i-1}
+    objstr[1][i] = string.format("%i %i ", num, objstr.off)
+    objstr[i+1] = content
+    objstr.off = objstr.off + #content
+  else
+    pdf[num] = {offset = pdf.file:seek()}
+    pdf.file:write(format('%i 0 obj\n', num))
+    pdf.file:write(content)
+    pdf.file:write'\nendobj\n'
+  end
   return num
 end
 local function delay(pdf, num, content, isfile)
@@ -84,25 +108,34 @@ local function getid(pdf)
   return id
 end
 local function trailer(pdf)
+  local linked = 0
+  for k,objstr in next, pdf.objstream do
+    objstr[1] = table.concat(objstr[1])
+    pdf:stream(objstr.objnum, string.format("/Type/ObjStm/N %i/First %i", #objstr-1, #objstr[1]), table.concat(objstr))
+  end
   local nextid = getid(pdf)
   local myoff = pdf.file:seek()
   pdf[nextid] = {offset = myoff}
-  local linked = 0
   local offsets = {}
   for i=1,nextid do
     local off = pdf[i].offset
     if off then
-      offsets[i+1] = pack(">I1I3I1", 1, off, 0)
+      offsets[i+1] = pack(">I1I3I2", 1, off, 0)
     else
-      offsets[linked+1] = pack(">I1I3I1", 0, i, 255)
-      linked = i
+      local objstr = pdf[i].objstr
+      if objstr then
+        offsets[i+1] = pack(">I1I3I2", 2, objstr, pdf[i].i)
+      else
+        offsets[linked+1] = pack(">I1I3I2", 0, i, 255)
+        linked = i
+      end
     end
   end
-  offsets[linked+1] = '\0\0\0\0\255'
+  offsets[linked+1] = '\0\0\0\0\255\255'
   pdf[nextid] = assigned
   -- TODO: Add an /ID according to 14.4
   local info = pdf.info and string.format("/Info %i 0 R", pdf.info) or ""
-  stream(pdf, nextid, format([[/Type/XRef/Size %i/W[1 3 1]/Root %i 0 R%s]], nextid+1, pdf.root, info), table.concat(offsets))
+  stream(pdf, nextid, format([[/Type/XRef/Size %i/W[1 3 2]/Root %i 0 R%s]], nextid+1, pdf.root, info), table.concat(offsets))
   pdf.file:write('startxref\n', myoff, '\n%%EOF')
 end
 local function close(pdf)
@@ -123,6 +156,7 @@ local pdfmeta = {
   indirect = indirect,
   stream = stream,
   newpage = pagetree.newpage,
+  reservepage = pagetree.reservepage,
   writepages = pagetree.write,
   delayed = delay,
   delayedstream = delayedstream,
@@ -133,7 +167,7 @@ pdfmeta.__index = pdfmeta
 local function open(filename)
   local file = io.open(filename, 'w')
   file:write"%PDF-X.X\n%🖋\n"
-  return setmetatable({file = file, version = '1.7', [0] = 0, pages = {}}, pdfmeta)
+  return setmetatable({file = file, version = '1.7', [0] = 0, pages = {}, objstream = {}}, pdfmeta)
 end
 return {
   open = open,

luametalatex-pdfe-deepcopy.lua

@@ -0,0 +1,90 @@
+local format = string.format
+local strip_floats = require'luametalatex-pdf-utils'.strip_floats
+local pdfe = pdfe
+local l = lpeg
+local regularchar = 1-l.S'\0\t\n\r\f ()<>[]{}/%#'
+local escapednamechar = l.P(1)/function(s)
+  return format("#%02X")
+end
+local nameescape = l.Cs(l.Cc'/' * (regularchar + escapednamechar)^0)
+local deepcopy_lookup deepcopy_lookup = {
+  function(_, pdf) -- 1: null
+    return 'null'
+  end,
+  function(_, pdf, b) -- 2: boolean
+    return b == 1 and 'true' or 'false'
+  end,
+  function(_, pdf, i) -- 3: integer
+    return format("%d", i)
+  end,
+  function(_, pdf, f) -- 4: number
+    return strip_floats(format("%f", f), "%.?0+[ %]]", "")
+  end,
+  function(_, pdf, name) -- 5: name
+    return nameescape:match(name)
+  end,
+  function(_, pdf, string, hex) -- 6: string
+    return hex and format("<%s>", string) or format("(%s)", string)
+  end,
+  function(references, pdf, array, size) -- 7: array
+    local a = {}
+    for i=1,size do
+      local type, value, detail = pdfe.getfromarray(array, i)
+      a[i] = deepcopy_lookup[type](references, pdf, value, detail)
+    end
+    return '[' .. table.concat(a, ' ') .. ']'
+  end,
+  function(references, pdf, dict, size) -- 8: dict
+    local a = {}
+    for i=1,size do
+      local key, type, value, detail = pdfe.getfromdictionary(dict, i)
+      a[2*i-1] = nameescape:match(key)
+      a[2*i] = deepcopy_lookup[type](references, pdf, value, detail)
+    end
+    return '<<' .. table.concat(a, ' ') .. '>>'
+  end,
+  nil, -- 9: stream (can only appear as a reference
+  function(references, pdf, ref, num)
+    local new = references[-num]
+    if not new then
+      new = pdf:getobj()
+      references[-num] = new
+      references[#references+1] = {ref, num}
+    end
+    return format("%i 0 R", new)
+  end,
+}
+
+local references = setmetatable({}, {__index = function(t, n)
+  local v = {}
+  t[n] = v
+  return v
+end})
+
+return function(file, id, pdf, type, value, detail)
+  local references = references[id]
+  local res = deepcopy_lookup[type](references, pdf, value, detail)
+  local i, r = 1, references[1]
+  while r do
+    local type, value, detail, more = pdfe.getfromreference(r[1])
+    if type == 9 then
+      local a,j = {}, 0
+      for i=1,more do
+        local key, type, value, detail = pdfe.getfromdictionary(detail, i)
+        if key == 'Length' then
+          j=2
+        else
+          a[2*i-1-j] = nameescape:match(key)
+          a[2*i-j] = deepcopy_lookup[type](references, pdf, value, detail)
+        end
+      end
+      pdf:stream(references[-r[2]], table.concat(a, ' '), value(false), false, true)
+    else
+      pdf:indirect(references[-r[2]], deepcopy_lookup[type](references, pdf, value, detail))
+    end
+    i = i+1
+    r = references[i]
+  end
+  for i=1,#references do references[i] = nil end
+  return res
+end

luametalatex-whatsits.lua

@@ -0,0 +1,96 @@
+local whatsit_id = node.id'whatsit'
+local whatsits = {
+  [0] = "open",
+        "write",
+        "close",
+        "special",
+        nil,
+        nil,
+        "save_pos",
+        "late_lua",
+        "user_defined",
+        nil,
+        nil,
+        nil,
+        nil,
+        nil,
+        nil,
+        nil,
+        "pdf_literal",
+        "pdf_refobj",
+        "pdf_annot",
+        "pdf_start_link",
+        "pdf_end_link",
+        "pdf_dest",
+        "pdf_action",
+        "pdf_thread",
+        "pdf_start_thread",
+        "pdf_end_thread",
+        "pdf_thread_data",
+        "pdf_link_data",
+        "pdf_colorstack",
+        "pdf_setmatrix",
+        "pdf_save",
+        "pdf_restore",
+}
+local whatsithandler = {}
+-- for i = 0,#whatsits do -- #whatsits isn't guaranteed to work because of the nil entries
+for i = 0,31 do
+  local v = whatsits[i]
+  if v then
+    whatsits[v] = i
+  end
+end
+function node.whatsits() return whatsits end
+function node.subtype(s) return type(s) == "string" and whatsits[s] or nil end
+
+local direct = node.direct
+local getsubtype = direct.getsubtype
+local properties = direct.get_properties_table()
+local tonode, todirect = direct.tonode, direct.todirect
+
+local function get_handler(n, subtype)
+  local props = properties[n]
+  return props and props.handle or whatsithandler[subtype or getsubtype(n)], props
+end
+local function new(name, handler)
+  assert(type(name) == 'string')
+  local subtype = whatsits[name]
+  if subtype then
+    if whatsithandler[subtype] then
+      texio.write_nl'WARNING: Overwriting default whatsit handler'
+    end
+  else
+    subtype = #whatsits + 1
+    whatsits[subtype] = name
+    whatsits[name] = subtype
+  end
+  whatsithandler[subtype] = handler
+  return subtype
+end
+
+-- TODO: Some fields might expect different values
+local function setwhatsitfield(n, name, value)
+  local props = properties[n]
+  if not props then
+    props = {}
+    properties[n] = props
+  end
+  props[name] = value
+end
+direct.setwhatsitfield = setwhatsitfield
+
+local function getwhatsitfield(n, name)
+  local props = properties[n]
+  return props and props[name]
+end
+direct.getwhatsitfield = getwhatsitfield
+
+-- TODO: Some fields might be nodes and therefore have to be converted
+function node.setwhatsitfield(n, ...) return setwhatsitfield(todirect(n), ...) end
+function node.getwhatsitfield(n, ...) return getwhatsitfield(todirect(n), ...) end
+
+return {
+  handler = get_handler,
+  new = new,
+}

luametalatex.lua

@@ -113,9 +113,10 @@ error[[CRITICAL: Initialization script not found]]
 -- error(msg)
 os.setenv("engine", status.luatex_engine)
 local ret_value
+local args = os.selfarg[1] and " \"" .. table.concat(os.selfarg, "\" \"") .. "\"" or ""
 if is_initex then
-  ret_value = os.execute(string.format("luametatex \"--lua=%s\" --arg0=\"%s\" \"%s\"", dir, os.selfarg[0], table.concat(os.selfarg, "\" \"")))
+  ret_value = os.execute(string.format("luametatex \"--lua=%s\" --arg0=\"%s\"%s", dir, os.selfarg[0], args))
 else
-  ret_value = os.execute(string.format("luametatex \"--fmt=%s\" \"--lua=%s\" --arg0=\"%s\" \"%s\"", format, dir, os.selfarg[0], table.concat(os.selfarg, "\" \"")))
+  ret_value = os.execute(string.format("luametatex \"--fmt=%s\" \"--lua=%s\" --arg0=\"%s\"%s", format, dir, os.selfarg[0], args))
 end
 os.exit(x)

luametaplain-init.lua

@@ -0,0 +1 @@
+luametalatex-init.lua

luametaplain.ini

@@ -0,0 +1,21 @@
+% tex-ini-files 2016-04-15: luatex.ini
+
+% Karl Berry, originally written 2008. Public domain.
+% PDF output by default.
+
+% Must be done first (as needs to 'tidy up')
+% \input luatexconfig.tex
+% Activate primitives
+\input luatexiniconfig.tex
+\begingroup
+% load-unicode-data needs \eTeXversion
+\catcode`\{=1
+\catcode`\}=2
+\global\chardef\eTeXversion=2
+\global\def\eTeXrevision{.2}
+\directlua{unhook_expl()}
+\endgroup
+\input load-unicode-data.tex
+\input etex.src
+\directlua{initialize_pdf_toks()}
+\dump

luametaplain.lua

@@ -0,0 +1 @@
+luametalatex.lua