EXPERIMENTAL: Implement file callbacks

luametalatex-callbacks.lua

@@ -1,98 +1,9 @@
--- Two callbacks are defined in other files: pre_dump in lateinit and find_fmt_file in init
-
-local read_tfm = font.read_tfm
-local font_define = font.define
-local callback_register = callback.register
-
-if status.ini_version then
-  callback_register('define_font', function(name, size)
-    local f = read_tfm(name, size)
-    if not f then return end
-    local id = font_define(f)
-    lua.prepared_code[#lua.prepared_code+1] = string.format("assert(%i == font.define(font.read_tfm(%q, %i)))", id, name, size)
-    return id
-  end)
-else
-  callback_register('define_font', function(name, size)
-    local f = read_tfm(name, size)
-    if not f then
-      tex.error(string.format("Font %q not found", name), "The requested font could't be loaded.\n\z
-        Are you sure that you passed the right name and\n\z
-        that the font is actually installed?")
-      return 0
-    end
-    return font.define(f)
-  end)
-end
-callback_register('find_log_file', function(name) return name end)
-do
-  local function normal_find_data_file(name)
-    return kpse.find_file(name, 'tex', true)
-  end
-  if status.ini_version then
-    function unhook_expl()
-      callback_register('find_data_file', normal_find_data_file)
-    end
-    callback_register('find_data_file', function(name)
-      if name == 'ltexpl.ltx' then
-        name = 'luametalatex-ltexpl-hook'
-      end
-      return normal_find_data_file(name)
-    end)
-  else
-    callback_register('find_data_file', normal_find_data_file)
-  end
-end
--- callback_register('read_data_file', function(name) error[[TODO]]return kpse.find_file(name, 'tex', true) end)
-callback_register('open_data_file', function(name)
-  local f = io.open(name, 'r')
-  return setmetatable({
-    reader = function()
-      local line = f:read()
-      return line
-    end,
-    close = function()error[[1]] return f:close() end,
-  }, {
-    __gc = function()f:close()end,
-  })
-end)
-callback_register('handle_error_hook', function()
-  repeat
-    texio.write_nl'? '
-    local line = io.read()
-    if not line then
-      tex.fatalerror'End of line encountered on terminal'
-    end
-    if line == "" then return 3 end
-    local first = line:sub(1,1):upper()
-    if first == 'H' then
-      texio.write(tex.gethelptext() or "Sorry, I don't know how to help in this situation.\n\z
-        Maybe you should try asking a human?")
-    elseif first == 'I' then
-      line = line:sub(2)
-      tex.runtoks(function()
-        tex.sprint(token.scan_token(), line)
-      end)
-      return 3
-    elseif first == 'Q' then texio.write'OK, entering \\batchmode...\n' return 0
-    elseif first == 'R' then texio.write'OK, entering \\nonstopmode...\n' return 1
-    elseif first == 'S' then texio.write'OK, entering \\scrollmode...\n' return 2
-    elseif first == 'X' then return -1
-    else
-      texio.write'Type <return> to proceed, S to scroll future error messages,\
-\z      R to run without stopping, Q to run quietly,\
-\z      I to insert something,\
-\z      H for help, X to quit.'
-    end
-  until false
-  return 3
-end)
-
 -- Now overwrite the callback functionality. Our system is based on the ssumption there there are
 -- no unknown callback names, just callbacks very unlikely to ever be called. That doesn't lead to
 -- good error checking, but we expect this to be overwritten by LaTeX anyway.
 
 local callback_find = callback.find
+local callback_register = callback.register
 local rawset = rawset
 local callbacks = setmetatable({}, {
   __index = function(cbs, name)

luametalatex-font-enc.lua

@@ -7,7 +7,7 @@ local lname = '/' * name / 1
 local namearray = lpeg.Ct('['*white^0*lpeg.Cg(lname*white^0, 0)^-1*(lname*white^0)^0*']')
 local encfile = white^0*lname*white^0*namearray*white^0*'def'*white^0*-1
 return function(filename)
-  local file <close> = readfile('enc', filename, nil, 'r')
+  local file <close> = readfile('enc', filename)
   local name, encoding = encfile:match(file())
   return encoding, name
 end

luametalatex-font-pk.lua

@@ -192,12 +192,9 @@ local function parse_commands(buf, off, t)
   until cmd == 245
   return off
 end
-return function(filename)
+return function(data)
-  local f = assert(io.open(filename, 'rb'))
-  local pk = f:read'a'
-  f:close()
   local res = {}
-  local off = parse_commands(pk, 1, res)
+  local off = parse_commands(data, 1, res)
   -- assert(off == #pk+1) -- TODO: Check that only fillup bytes follow
   return res
 end

luametalatex-font-resolve.lua

@@ -7,7 +7,7 @@ local callback_find = callback.find
 local old_font_define = font.define
 local old_addcharacters = font.addcharacters
 
-require'luametalatex-pdf-font-map'.mapfile(kpse.find_file('pdftex.map', 'map', true))
+require'luametalatex-pdf-font-map'.mapfile'pdftex.map'
 
 local all_fonts = {}
 font.fonts = all_fonts
@@ -16,9 +16,9 @@ function font.getfont(id)
 end
 
 local fontextensions = {
-  ttf = {"truetype", "truetype fonts",},
+  ttf = "truetype",
-  otf = {"opentype", "opentype fonts",},
+  otf = "opentype",
-  pfb = {"type1", "type1 fonts",},
+  pfb = "type1",
 }
 fontextensions.cff = fontextensions.otf
 local fontformats = {
@@ -60,39 +60,14 @@ function font.define(f)
     local entry = fontmap[f.name]
     if entry then
       local filename = entry[3]
-      local format = filename and filename:sub(-4, -4) == '.' and fontextensions[filename:sub(-3, -1)]
+      local format
-      if format then
+      if f.format == 'unknown' then
-        f.format = format[1]
+        f.format = filename and filename:sub(-4, -4) == '.' and fontextensions[filename:sub(-3, -1)] or 'type1'
-        f.filename = kpse.find_file(filename, format[2])
+      end
-        local encoding = entry[4]
+      f.filename = filename
-        if encoding then
+      f.encoding = entry[4]
-          f.encoding = kpse.find_file(encoding, 'enc files')
+      if entry[5] then
-        end
+        assert(special_parser:match(entry[5], 1, f))
-        if entry[5] then
-          assert(special_parser:match(entry[5], 1, f))
-        end
-      else
-        local done = false
-        for _, format in ipairs(fontformats) do
-          local filename = kpse.find_file(filename, format[2])
-          if filename then
-            f.filename = filename
-            f.format = format[1]
-            local encoding = entry[4]
-            if encoding then
-              f.encoding = kpse.find_file(encoding, 'enc files')
-            end
-            if entry[5] then
-              assert(special_parser:match(entry[5], 1, f))
-            end
-            done = true
-            break
-          end
-        end
-        if not done then
-          print('!', 'type3', require'inspect'(entry))
-          f.format = "type3"
-        end
       end
     else
       f.format = "type3"

luametalatex-font-tfm.lua

@@ -1,3 +1,5 @@
+local readfile = require'luametalatex-readfile'
+
 local upper_mask = (1<<20)-1<<44
 local shifted_sign = 1<<43
 local function scale(factor1, factor2)
@@ -141,13 +143,9 @@ local function parse_tfm(buf, i, size)
 end
 local basename = ((1-lpeg.S'\\/')^0*lpeg.S'\\/')^0*lpeg.C((1-lpeg.P'.tfm'*-1)^0)
 return function(name, size)
-  local filename = kpse.find_file(name, 'tfm', true)
+  local file <close> = readfile('tfm', name)
-  if not filename then return end
+  if not file then return end
-  local f = io.open(filename, 'rb')
+  local result = parse_tfm(file(), 1, size)
-  if not f then return end
-  local buf = f:read'*a'
-  f:close()
-  local result = parse_tfm(buf, 1, size)
   result.name = basename:match(name)
   return result
 end

luametalatex-font-vf.lua

@@ -1,3 +1,5 @@
+local readfile = require'luametalatex-readfile'
+
 local fontcmds = {
   [243] = ">I1I4I4I4BB",
   [244] = ">I2I4I4I4BB",
@@ -166,13 +168,9 @@ local function parse_vf(buf, i, size)
 end
 local basename = ((1-lpeg.S'\\/')^0*lpeg.S'\\/')^0*lpeg.C((1-lpeg.P'.tfm'*-1)^0)
 return function(name, size, must_exist)
-  local filename = kpse.find_file(name, 'vf', must_exist)
+  local file <close> = readfile('vf', name)
-  if not filename then return end
+  if not file then return end
-  local f = io.open(filename, 'rb')
+  local result = parse_vf(file(), 1, size)
-  if not f then return end
-  local buf = f:read'*a'
-  f:close()
-  local result = parse_vf(buf, 1, size)
   result.name = basename:match(name)
   return result
 end

luametalatex-lateinit.lua

@@ -13,7 +13,7 @@ pdf = {
   variable = {},
 }
 require'luametalatex-font-resolve' -- Replace font.define. Must be loaded before callbacks
-require'luametalatex-callbacks'
+require'luametalatex-basecallbacks'
 
 local functions = lua.getfunctionstable()
 -- I am not sure why this is necessary, but otherwise LuaMetaTeX resets

luametalatex-pdf-font-cff.lua

@@ -451,11 +451,6 @@ function myfunc(buf, i0, fontid, usedcids, encoding, trust_widths)
   end
   -- top.CharStrings = named_charstrings
   if not top.ROS then
-    -- if encoding == true and top.Encoding < 3 then
-      -- if not reencode and parsed_t1.Encoding == "StandardEncoding" then
-      --   reencode = kpse.find_file("8a.enc", "enc files")
-      -- end
-    -- end
     if encoding == true then -- Use the built-in encoding
       CharStrings = parse_encoding(buf, i0, top.Encoding, CharStrings)
     elseif encoding then
@@ -580,7 +575,7 @@ function myfunc(buf, i0, fontid, usedcids, encoding, trust_widths)
 end
 
 return function(filename, fontid, encoding) return function(fontdir, usedcids)
-  local file <close> = readfile('subset', filename, nil)
+  local file <close> = readfile('opentype', filename)
   local buf = file()
   local i = 1
   local magic = buf:sub(1, 4)

luametalatex-pdf-font-map.lua

@@ -40,7 +40,7 @@ local function mapfile(filename, operator)
   if not operator then
     operator, filename = optoperator:match(filename)
   end
-  local file <close> = readfile('map', filename, 'map', 'r')
+  local file <close> = readfile('map', filename)
   for line in file:lines() do mapline(line, operator) end
 end
 local function reset()

luametalatex-pdf-font-pk.lua

@@ -1,6 +1,7 @@
 local pk_global_resolution, pk_resolution_is_fixed
 local pdfvariable = pdf.variable
 
+local readfile = require'luametalatex-readfile'
 local read_pk = require'luametalatex-font-pk'
 local strip_floats = require'luametalatex-pdf-utils'.strip_floats
 return function(pdf, fontdir, usedcids)
@@ -13,7 +14,8 @@ return function(pdf, fontdir, usedcids)
     pk_resolution_is_fixed = pdfvariable.pkfixeddpi ~= 0
     kpse.init_prog("LUATEX", pk_global_resolution, pkmode ~= '' and pkmode or nil, nil) -- ?
   end
-  local pk = read_pk(kpse.find_file(fontdir.name, 'pk', pk_resolution_is_fixed and pk_global_resolution or (pk_global_resolution*fontdir.size/fontdir.designsize+.5)//1))
+  local f <close> = assert(readfile('pk', fontdir.name, pk_resolution_is_fixed and pk_global_resolution or (pk_global_resolution*fontdir.size/fontdir.designsize+.5)//1))
+  local pk = read_pk(f())
   local designsize = pk.designsize/1044654.326 -- 1044654.326=2^20*72/72.27 -- designsize in bp
   local hscale = 65536/pk.hppp / designsize -- 65291.158=2^16*72/72.27
   local vscale = 65536/pk.vppp / designsize -- 65291.158=2^16*72/72.27

luametalatex-pdf-font-t1.lua

@@ -6,7 +6,7 @@ local serializet2 = require'luametalatex-font-t2'
 local parseT1 = require'luametalatex-font-t1'
 local t1tot2 = require'luametalatex-font-t1tot2'
 return function(filename, reencode)
-  local file <close> = readfile('subset', filename, nil)
+  local file <close> = readfile('type1', filename)
   local parsed_t1 = parseT1(file())
   return function(f, usedcids)
     f.bbox = parsed_t1.FontBBox
@@ -40,7 +40,7 @@ return function(filename, reencode)
       -- LanguageGroup = parsed_t1.Private.LanguageGroup,
     }
     if not reencode and parsed_t1.Encoding == "StandardEncoding" then
-      reencode = kpse.find_file("8a.enc", "enc files")
+      reencode = '8a'
     end
     if reencode then
       parsed_t1.Encoding = require'luametalatex-font-enc'(reencode)

luametalatex-pdf-font-ttf.lua

@@ -69,7 +69,7 @@ local function readpostnames(buf, i, usedcids, encoding)
   return newusedcids
 end
 return function(filename, fontid, reencode)
-  local file <close> = readfile('subset', filename, nil)
+  local file <close> = readfile('truetype', filename)
   local buf = file()
   local magic, tables = sfnt.parse(buf, 1, fontid)
   if magic ~= "\0\1\0\0" then error[[Invalid TTF font]] end

luametalatex-pdf-image-pdf.lua

@@ -31,8 +31,16 @@ end
 
 local pdf_functions = {}
 
-local function open_pdfe(img)
+local function open_pdfe(img, f)
-  local file = pdfe.open(img.filepath)
+  local file
+  if f and f.file then
+    file = pdfe.openfile(f.file)
+    f.file = nil
+  elseif img.filedata then
+    file = pdfe.new(img.filedata, #img.filedata)
+  elseif img.filepath then
+    file = pdfe.open(img.filepath)
+  end
   do
     local userpassword = img.userpassword
     local ownerpassword = img.ownerpassword
@@ -51,8 +59,8 @@ local function open_pdfe(img)
     assert(false)
   end
 end
-function pdf_functions.scan(img)
+function pdf_functions.scan(img, f)
-  local file = open_pdfe(img)
+  local file = open_pdfe(img, f)
   img.pages = pdfe.getnofpages(file)
   img.page = img.page or 1
   if img.page > img.pages then

luametalatex-pdf-image-png.lua

@@ -1,4 +1,3 @@
-local readfile = require'luametalatex-readfile'
 local strip_floats = require'luametalatex-pdf-utils'.strip_floats
 
 local function ignore() end
@@ -7,17 +6,18 @@ local parse = setmetatable({
   -- PLTE = below,
   -- IDAT = below,
   -- IEND = below,
-  -- I'm not yet sure what to do about the following four color management chunks:
+  -- I originally thought about ignoring cHRM and gAMA (if you care about this stuff, you probably
-  -- These two will probably be ignored (if you care about this stuff, you probably
+  -- prefer an ICC profile anyway) but it was interesting to think about so it got implemented.
-  -- prefer an ICC profile anyway. Also especially cHRM requires some weird computations.)
+  -- But, gAMA is ONLY supported in combination with cHRM and not on it's own.
-  -- cHRM = TODO, -- ignore?
+  -- cHRM = below,
-  -- gAMA = TODO, -- ignore?
+  -- gAMA = below,
   -- iCCP is implemented, but profiles are not cached, so it might include the
   -- same profile many times
   -- iCCP = below,
   -- I would expect sRGB to be the most common, but it is a bit complicated because
-  -- PDF seems to require us to ship an actual ICC profile to support sRGB. Maybe later.
+  -- PDF seems to require us to ship an actual ICC profile to support sRGB. Luckily,
-  -- sRGB = TODO,
+  -- such a profile is part of TeXLive anyway.
+  -- sRGB = below,
   sBIT = ignore,
   bKGD = ignore, -- Background color. Ignored since we support transparency
   hIST = ignore, -- Color histogram
@@ -204,12 +204,8 @@ end
 
 local png_functions = {}
 
-function png_functions.scan(img)
+function png_functions.scan(img, f)
-  local file <close> = readfile('image', img.filepath, nil)
+  local buf = f()
-  if not file then
-    error[[PDF image could not be opened.]]
-  end
-  local buf = file()
   local t = run(buf, 1, #buf, 'IDAT')
   img.pages = 1
   img.page = 1
@@ -228,9 +224,9 @@ local intents = {[0]=
 }
 local function srgb_lookup(pfile, intent)
   if not srgb_colorspace then
-    local file <close> = readfile('silent', 'sRGB.icc.zlib', 'other binary files')
+    local file <close> = readfile('data', 'sRGB.icc')
     local profile = file()
-    local objnum = pfile:stream(nil, '/Filter/FlateDecode/N ' .. tostring(colortype & 2 == 2 and '3' or '1'), t.iCCP, nil, true)
+    local objnum = pfile:stream(nil, '/N 3', profile) -- FIXME: file stream
     srgb_colorspace = string.format('[/ICCBased %i 0 R]', objnum)
   end
   return objnum, intents[intent] or ''
@@ -258,11 +254,11 @@ local function rawimage(t, content)
 end
 
 function png_functions.write(pfile, img)
-  local file <close> = readfile('silent', img.filepath, nil)
+  local buf = img.filedata
-  if not file then
+  if not buf then
-    error[[PDF image could not be opened.]]
+    local f <close> = assert(io.open(img.filepath, 'rb'))
+    buf = f:read'a'
   end
-  local buf = file()
   local t = run(buf, 1, #buf, 'IEND')
   local colorspace
   local intent = ''

luametalatex-pdf-image.lua

@@ -1,3 +1,5 @@
+local readfile = require'luametalatex-readfile'
+
 local rawset = rawset
 local setdata = node.direct.setdata
 local nodenew = node.direct.new
@@ -61,14 +63,18 @@ local function scan(img)
     if m ~= meta then img = new(img) end
     real = real_images[img]
     if real.stream then error[[stream images are not yet supported]] end
-    assert(real.filename)
     -- TODO: At some point we should just take the lowercased extension
     local imagetype = real.filename:match'%.pdf$' and 'pdf'
                    or real.filename:match'%.png$' and 'png'
                    or error'Unsupported image format'
-    real.filepath = assert(kpse.find_file(real.filename), "Image not found")
     real.imagetype = imagetype
-    imagetypes[imagetype].scan(real)
+    local f <close>, path = assert(readfile('image', real.filename))
+    if f.file then
+      real.filepath = path
+    else
+      real.filedata = f.data
+    end
+    imagetypes[imagetype].scan(real, f)
     setmetatable(img, restricted_meta)
   end
   img.transform = img.transform or 0

luametalatex-readfile.lua

@@ -4,7 +4,22 @@ local find_file = kpse.find_file
 
 local callbacks = require'luametalatex-callbacks'
 
-local categories = { data = 1, map = 2, image = 3, subset = 4, font = 5, enc = 6, pdf_stream = 7, pdf_stream = 8, silent = 9}
+-- local categories = { data = 1, map = 2, image = 3, subset = 4, font = 5 -- , enc = 6, pdf_stream = 7, pdf_stream = 8, silent = 9}
+
+local our_callbacks = {
+  vf       = {'vf',             false, 'rb', 'find_vf_file_callback',       'read_vf_file_callback'},
+  tfm      = {'tfm',            false, 'rb', 'find_font_file_callback',     'read_font_file_callback'},
+  map      = {'map',            2,     'rb', 'find_map_file_callback',      'read_map_file_callback'},
+  enc      = {'enc',            false, 'rb', 'find_enc_file_callback',      'read_enc_file_callback'},
+  type1    = {'type1 fonts',    4,     'rb', 'find_type1_file_callback',    'read_type1_file_callback'},
+  truetype = {'truetype fonts', 4,     'rb', 'find_truetype_file_callback', 'read_truetype_file_callback'},
+  opentype = {'opentype fonts', 4,     'rb', 'find_opentype_file_callback', 'read_opentype_file_callback'},
+  pk       = {'pk',             4,     'rb', 'find_pk_file_callback',       'read_pk_file_callback'},
+  enc      = {'enc files',      false, 'rb', 'find_enc_file_callback',      'read_enc_file_callback'},
+  image    = {'tex',            3,     'rb', 'find_image_file_callback',    'read_image_file_callback'},
+  data     = {'tex',            1,     'rb', 'find_data_file_callback',     'read_data_file_callback'},
+}
+
 local start_categories = { [0] = '?', '(', '{', '<', '<', '<<' }
 local stop_categories  = { [0] = '?', ')', '}', '>', '>', '>>' }
 
@@ -15,34 +30,61 @@ local function stop_file(t)
   else
     write(stop_categories[t.category] or '')
   end
-  t.file:close()
+  if t.file then t.file:close() end
 end
 
-local meta = {
+local meta_file = {
   __close = stop_file,
   __call = function(t) return t.file:read'a' end,
   close = stop_file,
   lines = function(t, ...) return t.file:lines(...) end,
 }
-meta.__index = meta
+meta_file.__index = meta_file
 
-return function(category, name, kpse, mode)
+local meta_data = {
-  category = tonumber(category) or categories[category] or 0
+  __close = stop_file,
-  if kpse then
+  __call = function(t) return t.data end,
-    name = find_file(name, kpse)
+  close = stop_file,
-  end
+  lines = function(t, ...) return t.data:gmatch('([^\n]*)\n') end,
-  if not name then return name end
+}
-  local f, msg = io_open(name, mode or 'rb')
+meta_data.__index = meta_data
-  if f then
+
-    local cb = callbacks.start_file
+return function(kind, name, ...)
-    if cb then
+  local handle
-      cb(category, name)
+  local kind_info = our_callbacks[kind]
+  local msg
+  if kind_info then
+    local find_callback = callbacks[kind_info[4]]
+    if find_callback then
+      name, msg = find_callback(name, ...)
     else
-      local start_mark = start_categories[category]
+      name, msg = find_file(name, kind_info[1], ...)
-      if start_mark then
+    end
-        write(start_mark .. name)
+    if not name then return name, msg end
+    handle = {category = kind_info[2]}
+    local read_callback = callbacks[kind_info[5]]
+    if read_callback then
+      local success, data, size = read_callback(name)
+      if not success then return success, data end
+      if size < #data then data = data:sub(1,size) end
+      handle.data = data, data
+      setmetatable(handle, meta_data)
+    else
+      local f f, msg = io_open(name, kind_info[3])
+      if not f then return f, msg end
+      handle.file = f
+      setmetatable(handle, meta_file)
+    end
+    if handle.category then
+      local cb = callbacks.start_file
+      if cb then
+        cb(handle.category, name)
+      else
+        write(start_categories[handle.category] .. name)
       end
     end
+  else
+    error[[Unknown file]]
   end
-  return f and setmetatable({category = category, file = f}, meta), msg
+  return handle, name
 end