From aa00de1c9d0ef2597244f313b25b10e480847257 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marcel=20Fabian=20Kr=C3=BCger?= <zauguin@gmail.com>
Date: Sat, 4 Jul 2020 06:27:30 +0200
Subject: [PATCH] Load everything through bytecodes

---
 luametalatex-build-bytecode.lua |  28 +++++
 luametalatex-callbacks.lua      |  84 +++++++++++++++
 luametalatex-firstcode.lua      |  68 +-----------
 luametalatex-font-resolve.lua   |   3 +-
 luametalatex-init.lua           | 184 +++++++-------------------------
 luametalatex-lateinit.lua       | 131 +++++++++++++++++++++++
 6 files changed, 287 insertions(+), 211 deletions(-)
 create mode 100644 luametalatex-build-bytecode.lua
 create mode 100644 luametalatex-callbacks.lua
 create mode 100644 luametalatex-lateinit.lua

diff --git a/luametalatex-build-bytecode.lua b/luametalatex-build-bytecode.lua
new file mode 100644
index 0000000..ac05de4
--- /dev/null
+++ b/luametalatex-build-bytecode.lua
@@ -0,0 +1,28 @@
+local concat = table.concat
+local format = string.format
+local ioopen = io.open
+local assert = assert
+local ipairs = ipairs
+
+local _ENV = {}
+
+local first, later =
+  'local __hidden_local__package_preload__=package.preload',
+  '\n__hidden_local__package_preload__[%q]=function(...)%s\nend'
+
+first = first .. later
+
+local list = {}
+
+return function(t)
+  local length = #t
+  local tmpl = first
+  for i, mod in ipairs(t) do
+    local name, f = mod[1], assert(ioopen(mod[2], 'r'))
+    local data = f:read'a'
+    f:close()
+    list[i] = format(tmpl, name, data)
+    tmpl = later
+  end
+  return concat(list, nil, 1, length)
+end
diff --git a/luametalatex-callbacks.lua b/luametalatex-callbacks.lua
new file mode 100644
index 0000000..0f05ee6
--- /dev/null
+++ b/luametalatex-callbacks.lua
@@ -0,0 +1,84 @@
+-- Two callbacks are defined in other files: pre_dump in lateinit and find_fmt_file in init
+
+local read_tfm = font.read_tfm
+local font_define = font.define
+local callback_register = callback.register
+
+if status.ini_version then
+  callback_register('define_font', function(name, size)
+    local f = read_tfm(name, size)
+    if not f then return end
+    local id = font_define(f)
+    lua.prepared_code[#lua.prepared_code+1] = string.format("assert(%i == font.define(font.read_tfm(%q, %i)))", id, name, size)
+    return id
+  end)
+else
+  callback_register('define_font', function(name, size)
+    local f = read_tfm(name, size)
+    if not f then return end
+    return font.define(f)
+  end)
+end
+callback_register('find_log_file', function(name) return name end)
+do
+  local function normal_find_data_file(name)
+    return kpse.find_file(name, 'tex', true)
+  end
+  if status.ini_version then
+    function unhook_expl()
+      callback_register('find_data_file', normal_find_data_file)
+    end
+    callback_register('find_data_file', function(name)
+      if name == 'ltexpl.ltx' then
+        name = 'luametalatex-ltexpl-hook'
+      end
+      return normal_find_data_file(name)
+    end)
+  else
+    callback_register('find_data_file', normal_find_data_file)
+  end
+end
+-- callback_register('read_data_file', function(name) error[[TODO]]return kpse.find_file(name, 'tex', true) end)
+callback_register('open_data_file', function(name)
+  local f = io.open(name)
+  return setmetatable({
+    reader = function()
+      local line = f:read()
+      return line
+    end,
+    close = function()error[[1]] return f:close() end,
+  }, {
+    __gc = function()f:close()end,
+  })
+end)
+callback_register('handle_error_hook', function()
+  repeat
+    texio.write_nl'? '
+    local line = io.read()
+    if not line then
+      tex.fatalerror'End of line encountered on terminal'
+    end
+    if line == "" then return 3 end
+    local first = line:sub(1,1):upper()
+    if first == 'H' then
+      texio.write(tex.gethelptext() or "Sorry, I don't know how to help in this situation.\n\z
+        Maybe you should try asking a human?")
+    elseif first == 'I' then
+      line = line:sub(2)
+      tex.runtoks(function()
+        tex.sprint(token.scan_token(), line)
+      end)
+      return 3
+    elseif first == 'Q' then texio.write'OK, entering \\batchmode...\n' return 0
+    elseif first == 'R' then texio.write'OK, entering \\nonstopmode...\n' return 1
+    elseif first == 'S' then texio.write'OK, entering \\scrollmode...\n' return 2
+    elseif first == 'X' then return -1
+    else
+      texio.write'Type <return> to proceed, S to scroll future error messages,\
+\z      R to run without stopping, Q to run quietly,\
+\z      I to insert something,\
+\z      H for help, X to quit.'
+    end
+  until false
+  return 3
+end)
diff --git a/luametalatex-firstcode.lua b/luametalatex-firstcode.lua
index 15318b8..b75d93c 100644
--- a/luametalatex-firstcode.lua
+++ b/luametalatex-firstcode.lua
@@ -1,55 +1,4 @@
-local functions = lua.getfunctionstable()
--- I am not sure why this is necessary, but otherwise LuaMetaTeX resets
--- the functions table every time the getter is called
-function lua.get_functions_table() return functions end
-local set_lua = token.set_lua
--- There are two approaches to manage luafunctions ids without triggering
--- issues with ltluatex assigned numbers: Originally we assigned numbers
--- starting with 1, then switched to luatexbase ASAP and synchronised both
--- numbering schemes. But there is a useful quirk in ltluatex's luafunction
--- allocator: It only uses numbers upto 65535, so we can just use bigger
--- numbers. (This might have negative repercussins on performance because it
--- probably doesn't store the function in the array part of the Lua table.
--- Let's reconsider if this ever becomes a problem.
--- local new_luafunction = luatexbase.new_luafunction
-local predefined_luafunctions = status.ini_version and 65536 -- 1<<16  -- We start with 1<<16 + 1 (1<<16=65536 is reserved for luametalatex-local)
-local function new_luafunction(name)
-  if predefined_luafunctions then
-    predefined_luafunctions = predefined_luafunctions + 1
-    return predefined_luafunctions
-  else
-    error[[Here only preexisting luafunctions should be set]]
-  end
-end
-local undefined_cmd = token.command_id'undefined_cs'
 local lua_call_cmd = token.command_id'lua_call'
-local lua_value_cmd = token.command_id'lua_value'
-local lua_expandable_call_cmd = token.command_id'lua_expandable_call'
-function token.luacmd(name, func, ...)
-  local idx
-  local tok = token.create(name)
-  local cmd = tok.command
-  if cmd == lua_value_cmd then
-    idx = tok.mode
-  elseif cmd == lua_call_cmd then
-    idx = tok.mode
-  elseif cmd == lua_expandable_call_cmd then
-    idx = tok.mode
-  elseif ... == 'force' then
-    idx = new_luafunction(name)
-    set_lua(name, idx, select(2, ...))
-  elseif cmd == undefined_cmd then
-    idx = new_luafunction(name)
-    set_lua(name, idx, ...)
-  else
-    error(tok.cmdname)
-  end
-  if functions[idx] then
-    error[[Already defined]]
-  end
-  functions[idx] = func
-  return idx
-end
 local properties = node.direct.get_properties_table()
 node.direct.properties = properties
 function node.direct.get_properties_table()
@@ -154,6 +103,7 @@ token.luacmd("write", function(_, immediate) -- \write
   end
 end, "protected")
 
+local functions = lua.get_functions_table()
 token.luacmd("immediate", function() -- \immediate
   local next_tok = token.scan_token()
   if next_tok.command ~= lua_call_cmd then
@@ -166,22 +116,6 @@ end, "protected")
 --   local name = token.scan_string()
 --   print('Missing \\pdf' .. name)
 -- end
-if status.ini_version then
-  function fixupluafunctions()
-    return predefined_luafunctions
-  end
-else
-  function fixupluafunctions(i)
-    predefined_luafunctions = i
-  end
-  local prepared_code = lua.bytecode[1]
-  prepared_code()
-  lua.bytecode[1] = nil
-  -- function fixupluafunctions()
-    -- new_luafunction = luatexbase.new_luafunction
-    -- fixupluafunctions = nil
-  -- end
-end
 require'luametalatex-baseregisters'
 require'luametalatex-back-pdf'
 require'luametalatex-node-luaotfload'
diff --git a/luametalatex-font-resolve.lua b/luametalatex-font-resolve.lua
index 8c6c91f..08b53da 100644
--- a/luametalatex-font-resolve.lua
+++ b/luametalatex-font-resolve.lua
@@ -1,5 +1,6 @@
-local read_tfm = require'luametalatex-font-tfm'
+font.read_tfm = require'luametalatex-font-tfm'
 local read_vf = require'luametalatex-font-vf'
+font.read_vf = read_vf
 local fontmap = require'luametalatex-pdf-font-map'.fontmap
 local callback_find = callback.find
 
diff --git a/luametalatex-init.lua b/luametalatex-init.lua
index 7711f0c..94e67e4 100644
--- a/luametalatex-init.lua
+++ b/luametalatex-init.lua
@@ -1,7 +1,8 @@
 do
-  local ourpath = arg[0]:match('^%-%-lua=(.*[/\\])[^/\\]*%.lua$')
+  local ourpath = lua.startupfile:match('(.*[/\\])[^/\\]*%.lua$')
   kpse = assert(package.loadlib(ourpath .. 'kpse.so', 'luaopen_kpse'))()
 end
+local interaction
 do
   local arg0, progname
   for _, a in ipairs(arg) do
@@ -9,6 +10,17 @@ do
       progname = a:sub(12)
     elseif a:sub(1,7) == "--arg0=" then
       arg0 = a:sub(8)
+    elseif a:match'^%-%-?interaction=' then
+      local interaction_name = a:sub(a:find'='+1)
+      interaction = ({
+        batchmode=0,
+        nonstopmode=1,
+        scrollmode=2,
+        errorstopmode=3,
+      })[interaction_name]
+      if not interaction then
+        texio.write('term', string.format('Unknown interaction mode %q ignored.\n', interaction_name))
+      end
     end
   end
   kpse.set_program_name(arg0, progname)
@@ -25,149 +37,35 @@ package.searchers[2] = function(modname)
   return mod, filename
 end
 -- kpse.set_maketex("kpse_fmt_format", true)
-bit32 = require'luametalatex-bit32'
-kpse.init_prog("LUATEX", 400, "nexthi", nil)
-status.init_kpse = 1
+-- kpse.init_prog("LUATEX", 400, "nexthi", nil)
 require'luametalatex-init-config'
-status.safer_option = 0
-status.shell_escape = 0
-local read_tfm = require'luametalatex-font-tfm'
-local read_vf = require'luametalatex-font-vf'
-font.read_tfm = read_tfm
-font.read_vf = read_vf
 local callback_register = callback.register
-require'module'
-pdf = {
-  getfontname = function(id) -- No font sharing
-    return id
-  end,
-  variable = {},
-}
-require'luametalatex-font-resolve' -- Replace font.define
-
-local function base_define_font_cb(name, size)
-  local f = read_tfm(name, size)
-  if not f then return end
-  local id = font.define(f)
-  if status.ini_version then
-    lua.prepared_code[#lua.prepared_code+1] = string.format("assert(%i == font.define(font.read_tfm(%q, %i)))", id, name, size)
-  end
-  return id
-end
-callback_register('define_font', base_define_font_cb)
-callback_register('find_log_file', function(name) return name end)
-do
-  local function normal_find_data_file(name)
-    return kpse.find_file(name, 'tex', true)
-  end
-  if status.ini_version then
-    function unhook_expl()
-      callback_register('find_data_file', normal_find_data_file)
-    end
-    callback_register('find_data_file', function(name)
-      if name == 'ltexpl.ltx' then
-        name = 'luametalatex-ltexpl-hook'
-      end
-      return normal_find_data_file(name)
-    end)
-  else
-    callback_register('find_data_file', normal_find_data_file)
-  end
-end
-callback_register('read_data_file', function(name) error[[TODO]]return kpse.find_file(name, 'tex', true) end)
-callback_register('open_data_file', function(name)
-  local f = io.open(name)
-  return setmetatable({
-    reader = function()
-      local line = f:read()
-      return line
-    end,
-    close = function()error[[1]] return f:close() end,
-  }, {
-    __gc = function()f:close()end,
-  })
-end)
-callback_register('find_format_file', function(name) return kpse.find_file(name, 'fmt', true) end)
-callback_register('handle_error_hook', function()
-  repeat
-    texio.write_nl'? '
-    local line = io.read()
-    if not line then
-      tex.fatalerror'End of line encountered on terminal'
-    end
-    if line == "" then return 3 end
-    local first = line:sub(1,1):upper()
-    if first == 'H' then
-      texio.write(tex.gethelptext() or "Sorry, I don't know how to help in this situation.\n\z
-        Maybe you should try asking a human?")
-    elseif first == 'I' then
-      line = line:sub(2)
-      tex.runtoks(function()
-        tex.sprint(token.scan_token(), line)
-      end)
-      return 3
-    elseif first == 'Q' then texio.write'OK, entering \\batchmode...\n' return 0
-    elseif first == 'R' then texio.write'OK, entering \\nonstopmode...\n' return 1
-    elseif first == 'S' then texio.write'OK, entering \\scrollmode...\n' return 2
-    elseif first == 'X' then return -1
-    else
-      texio.write'Type <return> to proceed, S to scroll future error messages,\
-\z      R to run without stopping, Q to run quietly,\
-\z      I to insert something,\
-\z      H for help, X to quit.'
-    end
-  until false
-  return 3
-end)
-callback_register('pre_dump', function()
-  local prepared = lua.prepared_code
-  prepared[1] = string.format("fixupluafunctions(%i)", fixupluafunctions())
-  for i=0,0 do -- maybeFIXME: In practise only one language is preloaded in LuaTeX anyway
-  -- for i=0,tex.count[19] do -- Sometimes catches reserved language ids which are not used yet
-  -- for i=0,lang.new():id()-1 do -- lang.new():id() is always 0 in luametatex?!?
-    local l = lang.new(i)
-    local str = string.format("do \z
-      local l = lang.new(%i)\z
-      l:hyphenationmin(%i)\z
-      l:prehyphenchar(%i)\z
-      l:posthyphenchar(%i)\z
-      l:preexhyphenchar(%i)\z
-      l:postexhyphenchar(%i)",
-      i,
-      l:hyphenationmin(),
-      l:prehyphenchar(),
-      l:posthyphenchar(),
-      l:preexhyphenchar(),
-      l:postexhyphenchar())
-    local patterns = l:patterns()
-    local exceptions = l:hyphenation()
-    if patterns and exceptions then
-      str = string.format("%sl:patterns(%q)l:hyphenation(%q)end", str, patterns, exceptions)
-    elseif patterns then
-      str = string.format("%sl:patterns(%q)end", str, patterns)
-    elseif exceptions then
-      str = string.format("%sl:hyphenation(%q)end", str, exceptions)
-    else
-      str = str .. 'end'
-    end
-    prepared[#prepared+1] = str
-  end
-  for i=2,#prepared do
-    if type(prepared[i]) ~= 'string' then
-      prepared[i] = assert(prepared[i]())
-    end
-  end
-  lua.bytecode[1] = assert(load(table.concat(prepared, ' ')))
-end)
-function texconfig.init()
-  lua.bytecode[2]()
-  if not status.ini_version then
-    lua.bytecode[2] = nil
-  end
-end
+local build_bytecode
 if status.ini_version then
-  lua.prepared_code = {false}
-  local code = package.searchers[2]('luametalatex-firstcode')
-  if type(code) == "string" then error(string.format("Initialization code not found %s", code)) end
-  lua.bytecode[2] = code
+  local build_bytecode_mod = require'luametalatex-build-bytecode'
+  local preloaded_modules = {}
+  local old_searcher = package.searchers[2]
+  package.searchers[2] = function(name)
+    local mod, file = old_searcher(name)
+    if not file then return mod end -- Only works because we always return file when successful
+    preloaded_modules[#preloaded_modules+1] = {name, file}
+    return mod, file
+  end
+  function build_bytecode(str)
+    return load(build_bytecode_mod(preloaded_modules) .. "\nrequire'luametalatex-lateinit'(function()" .. str .. '\nend)', 'preloaded', 't')
+  end
+end
+
+callback_register('find_format_file', function(name) return kpse.find_file(name, 'fmt', true) end)
+function texconfig.init()
+  if interaction then
+    tex.setinteraction(interaction)
+  end
+  if build_bytecode then -- Effectivly if status.ini_version
+    require'luametalatex-lateinit'(build_bytecode)
+  else
+    local register = tex.count[262]+1
+    lua.bytecode[register]()
+    lua.bytecode[register] = nil
+  end
 end
diff --git a/luametalatex-lateinit.lua b/luametalatex-lateinit.lua
new file mode 100644
index 0000000..2af1370
--- /dev/null
+++ b/luametalatex-lateinit.lua
@@ -0,0 +1,131 @@
+local initex = status.ini_version
+
+if initex then
+  lua.prepared_code = {false}
+end
+
+bit32 = require'luametalatex-bit32' -- Why? And why so early?
+-- kpse.init_prog("LUATEX", 400, "nexthi", nil) -- ?
+status.init_kpse = 1 -- Why?
+status.safer_option = 0 -- compat
+status.shell_escape = 0 -- compat -- This is currently a lie.
+-- require'module' -- ???
+pdf = {
+  getfontname = function(id) -- No font sharing yet (TODO)
+    return id
+  end,
+  variable = {},
+}
+require'luametalatex-font-resolve' -- Replace font.define. Must be loaded before callbacks
+require'luametalatex-callbacks'
+
+local functions = lua.getfunctionstable()
+-- I am not sure why this is necessary, but otherwise LuaMetaTeX resets
+-- the functions table every time the getter is called
+function lua.get_functions_table() return functions end
+local set_lua = token.set_lua
+-- There are two approaches to manage luafunctions ids without triggering
+-- issues with ltluatex assigned numbers: Originally we assigned numbers
+-- starting with 1, then switched to luatexbase ASAP and synchronised both
+-- numbering schemes. But there is a useful quirk in ltluatex's luafunction
+-- allocator: It only uses numbers upto 65535, so we can just use bigger
+-- numbers. (This might have negative repercussins on performance because it
+-- probably doesn't store the function in the array part of the Lua table.
+-- Let's reconsider if this ever becomes a problem.
+-- local new_luafunction = luatexbase.new_luafunction
+local predefined_luafunctions = initex and 65536 -- 1<<16  -- We start with 1<<16 + 1 (1<<16=65536 is reserved for luametalatex-local)
+local new_luafunction
+function new_luafunction(name)
+  if predefined_luafunctions then
+    predefined_luafunctions = predefined_luafunctions + 1
+    return predefined_luafunctions
+  else
+    error[[Here only preexisting luafunctions should be set]]
+  end
+end
+local undefined_cmd = token.command_id'undefined_cs'
+local lua_call_cmd = token.command_id'lua_call'
+local lua_value_cmd = token.command_id'lua_value'
+local lua_expandable_call_cmd = token.command_id'lua_expandable_call'
+function token.luacmd(name, func, ...)
+  local idx
+  local tok = token.create(name)
+  local cmd = tok.command
+  if cmd == lua_value_cmd then
+    idx = tok.mode
+  elseif cmd == lua_call_cmd then
+    idx = tok.mode
+  elseif cmd == lua_expandable_call_cmd then
+    idx = tok.mode
+  elseif ... == 'force' then
+    idx = new_luafunction(name)
+    set_lua(name, idx, select(2, ...))
+  elseif cmd == undefined_cmd then
+    idx = new_luafunction(name)
+    set_lua(name, idx, ...)
+  else
+    error(tok.cmdname)
+  end
+  if functions[idx] then
+    error[[Already defined]]
+  end
+  functions[idx] = func
+  return idx
+end
+
+if initex then
+  local build_bytecode = nil -- To be filled
+  callback.register('pre_dump', function()
+    local prepared = lua.prepared_code
+    prepared[1] = string.format("fixupluafunctions(%i)", predefined_luafunctions)
+    for i=0,0 do -- maybeFIXME: In practise only one language is preloaded in LuaTeX anyway
+    -- for i=0,tex.count[19] do -- Sometimes catches reserved language ids which are not used yet
+    -- for i=0,lang.new():id()-1 do -- lang.new():id() is always 0 in luametatex?!?
+      local l = lang.new(i)
+      local str = string.format("do \z
+        local l = lang.new(%i)\z
+        l:hyphenationmin(%i)\z
+        l:prehyphenchar(%i)\z
+        l:posthyphenchar(%i)\z
+        l:preexhyphenchar(%i)\z
+        l:postexhyphenchar(%i)",
+        i,
+        l:hyphenationmin(),
+        l:prehyphenchar(),
+        l:posthyphenchar(),
+        l:preexhyphenchar(),
+        l:postexhyphenchar())
+      local patterns = l:patterns()
+      local exceptions = l:hyphenation()
+      if patterns and exceptions then
+        str = string.format("%sl:patterns(%q)l:hyphenation(%q)end", str, patterns, exceptions)
+      elseif patterns then
+        str = string.format("%sl:patterns(%q)end", str, patterns)
+      elseif exceptions then
+        str = string.format("%sl:hyphenation(%q)end", str, exceptions)
+      else
+        str = str .. 'end'
+      end
+      prepared[#prepared+1] = str
+    end
+    for i=2,#prepared do
+      if type(prepared[i]) ~= 'string' then
+        prepared[i] = assert(prepared[i]())
+      end
+    end
+    lua.bytecode[tex.count[262]+1] = build_bytecode(table.concat(prepared, '\n'))
+  end)
+  return function(f)
+    build_bytecode = f
+    return require'luametalatex-firstcode'
+  end
+else
+  function fixupluafunctions(i)
+    predefined_luafunctions = i
+    fixupluafunctions = nil
+  end
+  return function(f)
+    f()
+    return require'luametalatex-firstcode'
+  end
+end