Make PyCall.jl AOT-compilable (#651)

* Support PackageCompiler.jl; clear out global states

* Add scripts to run PackageCompiler

* Run tests with AOT-compiled PyCall in Travis

* Clear other global states

* Add aot/README.md

* Name Travis Jobs

* Add MacroTools to aot/Project.toml

* Add Pkg.activate in aot/precompile.jl

* Add Pkg.build("PyCall") in aot/compile.jl

Author: tkf

.gitignore

@@ -1,3 +1,7 @@
+aot/Manifest.toml
+aot/Project.toml
+aot/_julia_path
+aot/sys.*
 deps/deps.jl
 deps/PYTHON
 deps/build.log

.travis.yml

@@ -48,6 +48,19 @@ matrix:
 # https://github.com/pytest-dev/pytest/blob/799b72cf6f35c4c4c73797303d3f78c12a795f77/.travis.yml#L38-L52
 # https://github.com/pytest-dev/pytest/pull/3893
 
+    - &test-aot
+      name: "AOT (Julia: 1.0)"
+      language: julia
+      os: linux
+      env: PYTHON=python3
+      julia: 1.0
+      script:
+        - julia --color=yes -e 'using Pkg; Pkg.add("PackageCompiler")'
+        - julia --color=yes aot/compile.jl
+        - aot/runtests.sh
+      after_success: skip
+    - {<<: *test-aot, name: "AOT (Julia: nightly)", julia: nightly}
+
 after_success:
     - julia -e 'Pkg.add("Coverage"); using Coverage; Codecov.submit(process_folder())'
 notifications:

aot/README.md

@@ -0,0 +1,34 @@
+# Ahead-of-time compilation for PyCall
+
+This directory contains a set of scripts for testing compatibility of PyCall.jl
+with [PackageCompiler.jl](https://github.com/JuliaLang/PackageCompiler.jl).
+
+See `.travis.yml` for how it is actually used.
+
+## How to compile system image
+
+To create a system image with PyCall.jl, run `aot/compile.jl` (which
+is executable in *nix):
+
+```sh
+aot/compile.jl --color=yes
+JULIA=PATH/TO/CUSTOM/julia aot/compile.jl  # to specify a julia binary
+```
+
+Resulting system image is stored at `aot/sys.so`.
+
+## How to use compiled system image
+
+To use compiled system image, run `aot/julia.sh`, e.g.:
+
+```sh
+aot/julia.sh --compiled-modules=no --startup-file=no
+```
+
+Note that Julia binary used for compiling the system image is cached
+and automatically picked by `aot/julia.sh`.  You don't need to specify
+the Julia binary.
+
+Since Julia needs to re-compile packages when switching system images,
+it is recommended to pass `--compiled-modules=no` if you are using it
+in your machine with a standard Julia setup.

aot/compile.jl

@@ -0,0 +1,25 @@
+#!/bin/bash
+# -*- mode: julia -*-
+#=
+exec "${JULIA:-julia}" "$@" ${BASH_SOURCE[0]}
+=#
+
+using Pkg
+Pkg.activate(@__DIR__)
+Pkg.add("MacroTools")
+Pkg.develop(PackageSpec(path=dirname(@__DIR__)))
+Pkg.build("PyCall")
+Pkg.activate()
+
+using PackageCompiler
+sysout, _curr_syso = compile_incremental(
+    joinpath(@__DIR__, "Project.toml"),
+    joinpath(@__DIR__, "precompile.jl"),
+)
+
+pysysout = joinpath(@__DIR__, basename(sysout))
+cp(sysout, pysysout, force=true)
+
+write(joinpath(@__DIR__, "_julia_path"), Base.julia_cmd().exec[1])
+
+@info "System image: $pysysout"

aot/julia.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+thisdir="$(dirname "${BASH_SOURCE[0]}")"
+JULIA="${JULIA:-$(cat "$thisdir/_julia_path")}"
+exec "${JULIA}" --sysimage="$thisdir/sys.so" "$@"

aot/precompile.jl

@@ -0,0 +1,16 @@
+# Activate ./Project.toml.  Excluding `"@v#.#"` from `Base.LOAD_PATH`
+# to make compilation more reproducible.
+using Pkg
+empty!(Base.LOAD_PATH)
+append!(Base.LOAD_PATH, ["@", "@stdlib"])
+Pkg.activate(@__DIR__)
+
+# Manually invoking `__init__` to workaround:
+# https://github.com/JuliaLang/julia/issues/22910
+
+import MacroTools
+isdefined(MacroTools, :__init__) && MacroTools.__init__()
+
+using PyCall
+PyCall.__init__()
+PyCall.pyimport("sys")[:executable]

aot/runtests.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+thisdir="$(dirname "${BASH_SOURCE[0]}")"
+exec "$thisdir/julia.sh" --startup-file=no --color=yes -e '
+using Pkg
+Pkg.test("PyCall")
+'

src/pyinit.jl

@@ -26,6 +26,7 @@ const pyxrange = Ref{PyPtr}(0)
 
 function pyjlwrap_init()
     # PyMemberDef stores explicit pointers, hence must be initialized at runtime
+    empty!(pyjlwrap_members)  # for AOT
     push!(pyjlwrap_members, PyMemberDef(pyjlwrap_membername,
                                         T_PYSSIZET, sizeof_pyjlwrap_head, READONLY,
                                         pyjlwrap_doc),
@@ -125,6 +126,17 @@ function Py_Finalize()
 end
 
 function __init__()
+    # Clear out global states.  This is required only for PyCall
+    # AOT-compiled into system image.
+    _finalized[] = false
+    empty!(_namespaces)
+    empty!(eventloops)
+    empty!(npy_api)
+    empty!(pycall_gc)
+    empty!(pyexc)
+    empty!(pytype_queries)
+    empty!(permanent_strings)
+
     # sanity check: in Pkg for Julia 0.7+, the location of Conda can change
     # if e.g. you checkout Conda master, and we'll need to re-build PyCall
     # for something like pyimport_conda to continue working.