build-emacs-from-tar

#!/usr/bin/env ruby
#!nix-shell -i ruby dependencies.nix
#  Copyright © 2014-2023 David Caldwell
#  This program is free software: you can redistribute it and/or modify
#  it under the terms of the GNU General Public License as published by
#  the Free Software Foundation, either version 3 of the License, or
#  (at your option) any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with this program.  If not, see <http://www.gnu.org/licenses/>.

require 'optparse'
require 'fileutils'
require 'pathname'
require_relative 'build-dependencies'
require_relative 'verbose-shell'
Vsh = VerboseShell

def build_emacs(src_dir, dep_dir, out_name, options={})
  out_name = out_name + ".tar.bz2"
  puts "building emacs: #{src_dir} => #{out_name}"
  options[:cc] ||= "cc"
  options[:extra_cc_options] ||= ''

  ENV["PKG_CONFIG_PATH"]="#{dep_dir}/lib/pkgconfig:#{ENV["PKG_CONFIG_PATH"]}"
  ENV["PATH"]="#{dep_dir}/bin:#{ENV["PATH"]}"

  FileUtils.cd(src_dir) do
    min_os_flag = options[:min_os] ? "-mmacosx-version-min=#{options[:min_os]}" : ""
    configure_flags = options[:host] ? ["--host=#{options[:host]}", '--build=i686-apple-darwin'] : []
    parallel_flags = !options[:parallel] ? [] : ["-j", *(options[:parallel] == true ? [] : options[:parallel])]
    # This should be the default but isn't :-( http://debbugs.gnu.org/cgi/bugreport.cgi?bug=19850
    configure_flags += ['--enable-locallisppath=/Library/Application Support/Emacs/${version}/site-lisp:/Library/Application Support/Emacs/site-lisp']
    configure_flags += %W"--with-modules"
    # macOS has a low file descriptor limit for some reason and that breaks LSP stuff somehow.
    # https://en.liujiacai.net/2022/09/03/emacs-maxopenfiles/
    # https://github.com/caldwell/build-emacs/issues/127
    configure_flags += ['CFLAGS=-DFD_SETSIZE=10000 -DDARWIN_UNLIMITED_SELECT']

    ENV['CC']="#{options[:cc]} #{min_os_flag} #{options[:extra_cc_options]}"
    Vsh.system_trace(["CC=#{ENV['CC']}"])
    Vsh.system(*(%W"./configure --with-ns")+configure_flags+(options[:extra_configure_flags]||[]))
    Vsh.system(*(%W"make clean"))
    Vsh.system_noraise(*(%W"make")+parallel_flags) == 0 ||
    Vsh.system(        *(%W"make")+parallel_flags) # Try one more time if it fails! Emacs 29.4 always fails on the first `make` with `ranlib: file: libgnu.a(u64.o) has no symbols`
    Vsh.system(*(%W"make install"))

    strays = []
    ['nextstep/Emacs.app/Contents/MacOS/Emacs',
     *Dir['nextstep/Emacs.app/Contents/MacOS/bin/*',
          'nextstep/Emacs.app/Contents/MacOS/libexec/*']].select {|file| Vsh.capture(*%W"file #{file}") =~ /Mach-O/}
      .each {|exe|
      strays += copy_lib(exe, dep_dir, "nextstep/Emacs.app/Contents/MacOS/#{options[:libdir]}") # Install and adjust libs into the App.
    }
    report_strays strays

    FileUtils.cd('nextstep') { Vsh.system(*(%W"tar cjf #{out_name} Emacs.app")) }
  end
  Vsh.mv(File.join(src_dir, 'nextstep', out_name), out_name, :force => true)
  out_name
end

def with_writable_mode(file)
  old = File.stat(file).mode
  File.chmod(0775, file)
  yield
  File.chmod(old, file)
end

def copy_lib(exe, dep_dir, dest, options={})
  rel_path_to_dest = "@loader_path/" + Pathname.new(dest).relative_path_from(Pathname.new(exe).dirname).to_s.sub(/^\.$/,'')
  options[:rpath] ||= [];
  with_writable_mode(exe) {
    # remove our local build path from the id to leak as litle as possible (not that it really matters)
    Vsh.system(*%W"install_name_tool -id #{Pathname.new(exe).relative_path_from(Pathname.new(dest).dirname).to_s} #{exe}")
  }
  stray={ lib:[], path:[], exe:exe }
  stray[:path].concat(Vsh.capture(*%W"strings #{exe}").split("\n").select {|l| l.match(%r{/nix/store/}) })
  strays=[]
  Vsh.capture(*%W"otool -L #{exe}").split("\n").each do |line| # ex:   /Volumes/sensitive/src/build-emacs/brew/opt/gnutls/lib/libgnutls.30.dylib (compatibility version 37.0.0, current version 37.6.0)
    # HACK! I know we just added all that nice code to handle frameworks and rpaths (and
    # it works!), but it turns out codesign doesn't like this library. Perhaps because
    # it's named like a system library? Anyway, it appears to be compatible with the
    # actual system library, so lets just point to that, remove the rpath and be done.
    if %r{^\s+(?<cf>@rpath/CoreFoundation.framework/Versions/A/CoreFoundation)\s+} =~ line
      with_writable_mode(exe) {
        Vsh.system(*%W"install_name_tool -change #{cf} 	/System/Library/Frameworks/CoreFoundation.framework/Versions/A/CoreFoundation #{exe}") # Wheeee!
        Vsh.capture(*%W"otool -l #{exe}").split(/^(?=(?:Load command|Section))/m)
          .select {|c| /^\s*cmd LC_RPATH$/ =~ c}.map {|rp| /^\s*path\s+(?<path>.*)\s+\(offset[^)]+\)$/ =~ rp && path }
          .each {|rpath| Vsh.system(*%W"install_name_tool -delete_rpath #{rpath} #{exe}") }
      }
      next
    end
    (m,orig_dep,dep_base, dep_path,lib)=line.match(%r,^\s+((#{dep_dir}|@rpath)(/[^ ]+)+/(lib[^/ ]+))\s,).to_a
    (m,orig_dep,dep_base, dep_path,framework,lib)=line.match(%r,^\s+((#{dep_dir}|@rpath)(/[^ ]+)*/([^/ ]+\.framework)(/[^ ]+)+)\s,).to_a unless m
    if m
      # We have 2 boolean conditons (rp=rpath, fr=framework), so there are 4 cases we need to cover:
      #        Rename:   orig_dep                -> rel_path_to_dest/new_dep_lib                   Copy: orig_path                  -> dest                  Recurse: dest/new_dep_lib
      # -- --  #{dep_dir}/.../libx.dylib         -> @loader_path/#{dest:rel}/libx.dylib            #{dep_dir}/.../libx.dylib        -> #{dest}/libx.dylib    #{dest}/libx.dylib
      # -- fr  #{dep_dir}/.../X.framework/.../X  -> @loader_path/#{dest:rel}/X.framework/.../X     #{dep_dir}/.../X.framework       -> #{dest}/X.framework   #{dest}/X.framework/.../X
      # rp --  @rpath/.../libx.dylib             -> @loader_path/#{dest:rel}/libx.dylib            <resolved-rpath>/.../libx.dylib  -> #{dest}/libx.dylib    #{dest}/libx.dylib
      # rp fr  @rpath/.../X.framework/.../X      -> @loader_path/#{dest:rel}/X.framework/.../X     <resolved-rpath>/.../X.framework -> #{dest}/X.framework   #{dest}/X.framework/.../X
      if framework
        orig_path = File.join(dep_base, dep_path||"", framework)
        orig_lib = File.join(orig_path, lib)
        new_dep_lib = File.join(framework, lib)
      else
        orig_path = File.join(dep_base, dep_path, lib)
        orig_lib = orig_path
        new_dep_lib = lib
      end

      if dep_base == "@rpath"
        # Accumulating our rpaths here isn't technically correct--If some random binary
        # down the chain has an rpath this makes it pollute our lookups from then
        # on. Practically it should be ok since we are currently pulling from Nix and
        # everything _should_ be sharing the same stuff anyway.
        options[:rpath] += Vsh.capture(*%W"otool -l #{exe}").split(/^(?=(?:Load command|Section))/m)
                             .select {|c| /^\s*cmd LC_RPATH$/ =~ c}.map {|rp| /^\s*path\s+(?<path>.*)\s+\(offset[^)]+\)$/ =~ rp && path }
        rpath = options[:rpath].select {|p| File.exist?(orig_dep.sub(/@rpath/, p)) }.first or raise "Can't resolve rpath #{orig_dep} in #{rpaths}"
        orig_path = orig_path.sub(/@rpath/, rpath)
        orig_lib = orig_lib.sub(/@rpath/, rpath)
      end

      with_writable_mode(exe) {
        Vsh.system(*%W"install_name_tool -change #{orig_dep} #{File.join(rel_path_to_dest, new_dep_lib)} #{exe}") # Point to where we're about to copy the lib
      }
      unless File.exist?(File.join(dest, File.basename(orig_path)))
        Vsh.mkdir_p(dest)
        Vsh.cp_r(orig_path, dest)
        strays.concat copy_lib(File.join(dest, new_dep_lib), dep_dir, dest, options) # Copy lib's deps, too
      end
    elsif !line.match(%r{^(?:\s+(?:/System/|@executable_path/|/usr/lib/libSystem\.\w+\.dylib|/usr/lib/libresolv.\w+.dylib|#{Regexp.escape(File.basename(dest))}))|^#{Regexp.escape(exe)}:})
      stray[:lib].push(line)
    end
  end
  return strays.concat(stray[:lib].any? || stray[:path].any? ? [stray] : [])
end

def prepare_extra_deps(dep_dir, out_name)
  extra_source = "#{out_name}-extra-source"
  Vsh.rm_rf extra_source
  build_dep = BuildDependencies.new(dep_dir)
  build_dep.ensure()
  build_dep.export_sources(extra_source)
  Vsh.system(*(%W"tar cf #{extra_source}.tar #{extra_source}"))
  "#{extra_source}.tar"
end

def report_strays(stray)
  normal = "\e[0m"
  orange = "\e[1;38;5;214;40m"
  red    = "\e[1;38;5;196;40m"
  white  = "\e[97;40m"
  msg =  stray.select {|s| s[:path].any? }.map {|s| { severity: :warn, msg: "#{s[:exe]} contains paths to the nix store", info: s[:path] } }
  msg += stray.select {|s| s[:lib].any?  }.map {|s| { severity: ENV["IN_NIX_SHELL"] ? :error : :warn,
                                                      msg: "#{s[:exe]} contains non-encapsulated libs", info: s[:lib] } }
  msg.each do |m|
    printf("%s#{white} %s:#{normal}\n", {warn: "#{orange}Warning:", error: "#{red}Error:"}[m[:severity]], m[:msg])
    m[:info].each {|i| puts "  #{i}" }
  end
  STDOUT.flush # Otherwise the raise on the next line comes out way before the actual errors (stderr is always unbuffered)
  raise "💩 Fatal: Non-encapsulated libs" if msg.any? {|m| m[:severity] == :error}
end

if File.directory?('/nix/store') && !ENV["IN_NIX_SHELL"]
  # If we have a nix install and we aren't already in a nix-shell, then relaunch inside one
  #
  # The docs want us to use `--run` but that requires stringifying our arguments
  # safely which is gross. So instead we abuse nix-shell's ability to run as an
  # interpretter to pass in our args in array form by passing our script to it
  # as if it was called from a #! line. This requires the 2nd line of our file
  # to be the `#!nix-shell line` described in the docs. Our dependencies are
  # specified there instead of here, unfortunately, but at least we safely pass
  # our entire command line through.
  puts "Relaunching inside a clean Nix environment..."
  STDOUT.flush # In CI, stdout is a pipe and not line buffered, gotta get our msg out before exec()
  Vsh.system_trace("exec", "nix-shell", $0, *ARGV)
  exec('nix-shell', $0, *ARGV)
end

arch=`uname -m`.chomp.to_sym
parallel=false
disable_deps=false
extra_rev = ''
add_configure_flags = []

(opts=OptionParser.new do |opts|
  opts.banner = "Usage:\n\t#{$0} <SOURCE_TARBALL> <KIND> [options]"
  opts.on("-v", "--verbose", "Turn up the verbosity") { |v| Vsh.verbose = true }
  opts.on("-a", "--arch ARCH", [:i386, :x86_64, :arm64], "Compile for ARCH instead of #{arch.to_s}") { |a| arch = a }
  opts.on("-j", "--parallel [PROCS]", "Compile in parallel using PROCS processes") { |p| parallel = p || true }
  opts.on(      "--extra-rev REV", "Add an extra -REV to the version") { |r| extra_rev = r }
  opts.on(      "--no-deps", "Don't attempt to get any extra libraries") { |b| disable_deps = true }
  opts.on("-A", "--add-configure-flags FLAGS", "Additional configure flags") { |c| add_configure_flags.push(c) }
  opts.on_tail("-h", "--help") { puts opts; exit }
end).parse!

source_tar = ARGV.shift || opts.abort("Missing <SOURCE_TARBALL>\n\n"+opts.help)
kind       = ARGV.shift || opts.abort("Missing <KIND>\n\n"+opts.help)

label = kind == 'pretest' ? 'pretest-' : ''
version = source_tar =~ %r{^(?:.*/)?emacs-(.*)\.tar} && $1 || throw("couldn't parse version from #{source_tar}")
trunk = !!(version =~ /^\d{4}-\d{2}-\d{2}/)

src_dir = 'emacs-source'
dep_dir = File.expand_path("dep")

os_maj_version = `sw_vers -productVersion`.chomp.sub(/^(\d+\.\d+)\.\d+/,'\1')
os_maj_version = $1 if os_maj_version =~ /^(\d+)\./ && $1.to_i > 10 # After 10.15.x Apple moved to 11.x, 12.x, etc.

options = arch == :i386    ? { :cc => 'i686-apple-darwin10-gcc-4.2.1',    :host => 'i686-apple-darwin',    :min_os => '10.5' } :
                             { }
options[:min_os] =  '10.7' if os_maj_version == '10.8'
options[:min_os] =  '10.5' if os_maj_version == '10.6' && arch == :x86_64
options[:min_os] =  '10.6' if trunk && options[:min_os] == '10.5'
options[:min_os] =  '11'   if os_maj_version == '12' && arch == :arm64 # Hack around thoughtless upgrade of the build machine to macOS 12 🤦
options[:extra_configure_flags] ||= []
options[:extra_configure_flags] += %w"--with-jpeg=no --with-png=no --with-gif=no --with-tiff=no" if os_maj_version == '10.6'
options[:extra_configure_flags] += %w"--with-gnutls=no" if disable_deps
options[:extra_configure_flags] += %w"CFLAGS=-DNSTextAlignmentRight=NSRightTextAlignment" if os_maj_version.to_f < 10.12
options[:extra_configure_flags] += add_configure_flags if ! add_configure_flags.empty?
options[:parallel] = parallel if parallel
options[:libdir] = 'lib-' + arch.to_s + '-' + (options[:min_os] || os_maj_version).to_s.gsub('.','_') # see similar gsub in combine-and-package

out_name = "Emacs-#{label}#{version}#{extra_rev}-#{options[:min_os] || os_maj_version}-#{arch.to_s}"

if ENV["IN_NIX_SHELL"] # If we're in a nix-shell then get our deps from there
  dep_dir = '/nix/store'
else
  extra_source = prepare_extra_deps dep_dir, out_name unless disable_deps
end

Vsh.rm_rf src_dir
Vsh.mkdir_p src_dir
FileUtils.cd(src_dir) do
  Vsh.system(*%W'tar xf #{"../"+source_tar} --strip-components=1')
end

binary = build_emacs src_dir, dep_dir, out_name, options

puts "Built #{binary}, #{extra_source||''}"