From 0fddf6f7fc50fe39ad1d34f37030560a5040dccb Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Tue, 22 Nov 2022 18:22:59 -0500 Subject: [PATCH 01/19] Add PackageCompiler.jl to PySR project --- pysr/julia_helpers.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/pysr/julia_helpers.py b/pysr/julia_helpers.py index b0ec30d24..64017175c 100644 --- a/pysr/julia_helpers.py +++ b/pysr/julia_helpers.py @@ -252,10 +252,17 @@ def _add_sr_to_julia_project(Main, io_arg): ) Main.clustermanagers_spec = Main.PackageSpec( name="ClusterManagers", - url="https://github.com/JuliaParallel/ClusterManagers.jl", - rev="14e7302f068794099344d5d93f71979aaf4fbeb3", + rev="v0.4.2", + ) + Main.packagecompiler_spec = Main.PackageSpec( + name="PackageCompiler", + rev="v2.1.0", + ) + Main.eval( + "Pkg.add([" + + ", ".join(["sr_spec", "clustermanagers_spec", "packagecompiler_spec"]) + + f"], {io_arg})" ) - Main.eval(f"Pkg.add([sr_spec, clustermanagers_spec], {io_arg})") def _escape_filename(filename): From c865ccba64bce9e2f048e672cb6e0a843e0d9b62 Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Tue, 22 Nov 2022 18:23:14 -0500 Subject: [PATCH 02/19] Add function to create simple sysimage --- pysr/package_compiler.py | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 pysr/package_compiler.py diff --git a/pysr/package_compiler.py b/pysr/package_compiler.py new file mode 100644 index 000000000..f7a5b92b5 --- /dev/null +++ b/pysr/package_compiler.py @@ -0,0 +1,36 @@ +"""Functions to create a sysimage for PySR.""" + +from pathlib import Path +import numpy as np + +from . import PySRRegressor +from .julia_helpers import init_julia + + +def create_sysimage(): + """Create a PackageCompiler.jl sysimage from a simple PySR run.""" + sysimage_name = "pysr.so" + + # Example dataset: + X = 2 * np.random.randn(100, 5) + y = 2.5382 * np.cos(X[:, 3]) + X[:, 0] ** 2 - 0.5 + + model = PySRRegressor( + unary_operators=["cos"], + procs=8, + niterations=2, + populations=4, + verbosity=0, + progress=False, + ) + model.fit(X, y) + + from julia import Main + + cur_project_dir = Main.eval("splitdir(dirname(Base.active_project()))[1]") + from julia import PackageCompiler + + PackageCompiler.create_sysimage( + ["SymbolicRegression"], + sysimage_path=str(Path(cur_project_dir) / sysimage_name), + ) From bc506551c91d6f3c2b6d4640163992200545fd8a Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Tue, 22 Nov 2022 18:23:27 -0500 Subject: [PATCH 03/19] Add `create_sysimage` to `__init__` --- pysr/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pysr/__init__.py b/pysr/__init__.py index e303becb2..f6e5c9894 100644 --- a/pysr/__init__.py +++ b/pysr/__init__.py @@ -11,3 +11,4 @@ from .feynman_problems import Problem, FeynmanProblem from .export_jax import sympy2jax from .export_torch import sympy2torch +from .package_compiler import create_sysimage From 91249346327c377a98e97389a07f4fb57274d78f Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Tue, 22 Nov 2022 18:26:49 -0500 Subject: [PATCH 04/19] Put `pysr.so` into correct directory --- pysr/package_compiler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysr/package_compiler.py b/pysr/package_compiler.py index f7a5b92b5..4058b9292 100644 --- a/pysr/package_compiler.py +++ b/pysr/package_compiler.py @@ -27,7 +27,7 @@ def create_sysimage(): from julia import Main - cur_project_dir = Main.eval("splitdir(dirname(Base.active_project()))[1]") + cur_project_dir = Main.eval("dirname(Base.active_project())") from julia import PackageCompiler PackageCompiler.create_sysimage( From f7a8f6c309643957ee951974292f72bc156b3439 Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Mon, 28 Nov 2022 12:27:05 -0500 Subject: [PATCH 05/19] Add option to create sysimage to `install()` --- pysr/julia_helpers.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pysr/julia_helpers.py b/pysr/julia_helpers.py index 64017175c..bb07cb2d4 100644 --- a/pysr/julia_helpers.py +++ b/pysr/julia_helpers.py @@ -7,6 +7,7 @@ from julia.api import JuliaError from .version import __version__, __symbolic_regression_jl_version__ +from .package_compiler import create_sysimage juliainfo = None julia_initialized = False @@ -65,7 +66,7 @@ def _get_io_arg(quiet): return io_arg -def install(julia_project=None, quiet=False, precompile=None): # pragma: no cover +def install(julia_project=None, quiet=False, precompile=None, compile=False): # pragma: no cover """ Install PyCall.jl and all required dependencies for SymbolicRegression.jl. @@ -97,10 +98,12 @@ def install(julia_project=None, quiet=False, precompile=None): # pragma: no cov Main.eval("using Pkg") Main.eval(f"Pkg.instantiate({io_arg})") - if precompile: Main.eval(f"Pkg.precompile({io_arg})") + if compile: + create_sysimage(julia_project=julia_project, quiet=quiet) + if not quiet: warnings.warn( "It is recommended to restart Python after installing PySR's dependencies," From 0e576734bfcc981b76d293acc0296a67c76783f1 Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Mon, 28 Nov 2022 12:27:21 -0500 Subject: [PATCH 06/19] Clean up sysimage creation --- pysr/package_compiler.py | 26 ++++---------------------- 1 file changed, 4 insertions(+), 22 deletions(-) diff --git a/pysr/package_compiler.py b/pysr/package_compiler.py index 4058b9292..22cd4b261 100644 --- a/pysr/package_compiler.py +++ b/pysr/package_compiler.py @@ -7,30 +7,12 @@ from .julia_helpers import init_julia -def create_sysimage(): +def create_sysimage(sysimage_name="pysr.so", julia_project=None, quiet=False): """Create a PackageCompiler.jl sysimage from a simple PySR run.""" - sysimage_name = "pysr.so" - - # Example dataset: - X = 2 * np.random.randn(100, 5) - y = 2.5382 * np.cos(X[:, 3]) + X[:, 0] ** 2 - 0.5 - - model = PySRRegressor( - unary_operators=["cos"], - procs=8, - niterations=2, - populations=4, - verbosity=0, - progress=False, - ) - model.fit(X, y) - - from julia import Main + Main = init_julia(julia_project=julia_project, quiet=quiet) cur_project_dir = Main.eval("dirname(Base.active_project())") + sysimage_path = str(Path(cur_project_dir) / sysimage_name) from julia import PackageCompiler - PackageCompiler.create_sysimage( - ["SymbolicRegression"], - sysimage_path=str(Path(cur_project_dir) / sysimage_name), - ) + PackageCompiler.create_sysimage(["SymbolicRegression"], sysimage_path=sysimage_path) From 494d282c36e95ef38bf30dba5321e2b9a66afcb3 Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Mon, 28 Nov 2022 12:33:52 -0500 Subject: [PATCH 07/19] Move installation utilities to compilation module --- pysr/__init__.py | 2 +- pysr/julia_helpers.py | 78 -------------------------------- pysr/package_compiler.py | 97 ++++++++++++++++++++++++++++++++++++++-- pysr/sr.py | 2 +- 4 files changed, 95 insertions(+), 84 deletions(-) diff --git a/pysr/__init__.py b/pysr/__init__.py index f6e5c9894..2b7c64601 100644 --- a/pysr/__init__.py +++ b/pysr/__init__.py @@ -7,7 +7,7 @@ best_callable, best_row, ) -from .julia_helpers import install +from .package_compiler import install from .feynman_problems import Problem, FeynmanProblem from .export_jax import sympy2jax from .export_torch import sympy2torch diff --git a/pysr/julia_helpers.py b/pysr/julia_helpers.py index bb07cb2d4..b281f426f 100644 --- a/pysr/julia_helpers.py +++ b/pysr/julia_helpers.py @@ -7,7 +7,6 @@ from julia.api import JuliaError from .version import __version__, __symbolic_regression_jl_version__ -from .package_compiler import create_sysimage juliainfo = None julia_initialized = False @@ -66,51 +65,6 @@ def _get_io_arg(quiet): return io_arg -def install(julia_project=None, quiet=False, precompile=None, compile=False): # pragma: no cover - """ - Install PyCall.jl and all required dependencies for SymbolicRegression.jl. - - Also updates the local Julia registry. - """ - import julia - - _julia_version_assertion() - # Set JULIA_PROJECT so that we install in the pysr environment - processed_julia_project, is_shared = _process_julia_project(julia_project) - _set_julia_project_env(processed_julia_project, is_shared) - - if precompile == False: - os.environ["JULIA_PKG_PRECOMPILE_AUTO"] = "0" - - julia.install(quiet=quiet) - Main, init_log = init_julia(julia_project, quiet=quiet, return_aux=True) - io_arg = _get_io_arg(quiet) - - if precompile is None: - precompile = init_log["compiled_modules"] - - if not precompile: - Main.eval('ENV["JULIA_PKG_PRECOMPILE_AUTO"] = 0') - - if is_shared: - # Install SymbolicRegression.jl: - _add_sr_to_julia_project(Main, io_arg) - - Main.eval("using Pkg") - Main.eval(f"Pkg.instantiate({io_arg})") - if precompile: - Main.eval(f"Pkg.precompile({io_arg})") - - if compile: - create_sysimage(julia_project=julia_project, quiet=quiet) - - if not quiet: - warnings.warn( - "It is recommended to restart Python after installing PySR's dependencies," - " so that the Julia environment is properly initialized." - ) - - def _import_error(): return """ Required dependencies are not installed or built. Run the following code in the Python REPL: @@ -246,28 +200,6 @@ def init_julia(julia_project=None, quiet=False, julia_kwargs=None, return_aux=Fa return Main -def _add_sr_to_julia_project(Main, io_arg): - Main.eval("using Pkg") - Main.sr_spec = Main.PackageSpec( - name="SymbolicRegression", - url="https://github.com/MilesCranmer/SymbolicRegression.jl", - rev="v" + __symbolic_regression_jl_version__, - ) - Main.clustermanagers_spec = Main.PackageSpec( - name="ClusterManagers", - rev="v0.4.2", - ) - Main.packagecompiler_spec = Main.PackageSpec( - name="PackageCompiler", - rev="v2.1.0", - ) - Main.eval( - "Pkg.add([" - + ", ".join(["sr_spec", "clustermanagers_spec", "packagecompiler_spec"]) - + f"], {io_arg})" - ) - - def _escape_filename(filename): """Turn a path into a string with correctly escaped backslashes.""" str_repr = str(filename) @@ -309,16 +241,6 @@ def _load_cluster_manager(Main, cluster_manager): return Main.eval(f"addprocs_{cluster_manager}") -def _update_julia_project(Main, is_shared, io_arg): - try: - if is_shared: - _add_sr_to_julia_project(Main, io_arg) - Main.eval("using Pkg") - Main.eval(f"Pkg.resolve({io_arg})") - except (JuliaError, RuntimeError) as e: - raise ImportError(_import_error()) from e - - def _load_backend(Main): try: # Load namespace, so that various internal operators work: diff --git a/pysr/package_compiler.py b/pysr/package_compiler.py index 22cd4b261..9c9acbd32 100644 --- a/pysr/package_compiler.py +++ b/pysr/package_compiler.py @@ -1,15 +1,104 @@ """Functions to create a sysimage for PySR.""" +import os from pathlib import Path +import warnings + import numpy as np +from julia.api import JuliaError -from . import PySRRegressor -from .julia_helpers import init_julia +from .version import __symbolic_regression_jl_version__ +from .julia_helpers import ( + init_julia, + _julia_version_assertion, + _set_julia_project_env, + _get_io_arg, + _process_julia_project, + _import_error, +) -def create_sysimage(sysimage_name="pysr.so", julia_project=None, quiet=False): - """Create a PackageCompiler.jl sysimage from a simple PySR run.""" +def _add_sr_to_julia_project(Main, io_arg): + Main.eval("using Pkg") + Main.sr_spec = Main.PackageSpec( + name="SymbolicRegression", + url="https://github.com/MilesCranmer/SymbolicRegression.jl", + rev="v" + __symbolic_regression_jl_version__, + ) + Main.clustermanagers_spec = Main.PackageSpec( + name="ClusterManagers", + rev="v0.4.2", + ) + Main.packagecompiler_spec = Main.PackageSpec( + name="PackageCompiler", + rev="v2.1.0", + ) + Main.eval( + "Pkg.add([" + + ", ".join(["sr_spec", "clustermanagers_spec", "packagecompiler_spec"]) + + f"], {io_arg})" + ) + + +def _update_julia_project(Main, is_shared, io_arg): + try: + if is_shared: + _add_sr_to_julia_project(Main, io_arg) + Main.eval("using Pkg") + Main.eval(f"Pkg.resolve({io_arg})") + except (JuliaError, RuntimeError) as e: + raise ImportError(_import_error()) from e + + +def install( + julia_project=None, quiet=False, precompile=None, compile=False +): # pragma: no cover + """ + Install PyCall.jl and all required dependencies for SymbolicRegression.jl. + + Also updates the local Julia registry. + """ + import julia + + _julia_version_assertion() + # Set JULIA_PROJECT so that we install in the pysr environment + processed_julia_project, is_shared = _process_julia_project(julia_project) + _set_julia_project_env(processed_julia_project, is_shared) + + if precompile == False: + os.environ["JULIA_PKG_PRECOMPILE_AUTO"] = "0" + julia.install(quiet=quiet) + Main, init_log = init_julia(julia_project, quiet=quiet, return_aux=True) + io_arg = _get_io_arg(quiet) + + if precompile is None: + precompile = init_log["compiled_modules"] + + if not precompile: + Main.eval('ENV["JULIA_PKG_PRECOMPILE_AUTO"] = 0') + + if is_shared: + # Install SymbolicRegression.jl: + _add_sr_to_julia_project(Main, io_arg) + + Main.eval("using Pkg") + Main.eval(f"Pkg.instantiate({io_arg})") + if precompile: + Main.eval(f"Pkg.precompile({io_arg})") + + if compile: + create_sysimage(julia_project=julia_project, quiet=quiet) + + if not quiet: + warnings.warn( + "It is recommended to restart Python after installing PySR's dependencies," + " so that the Julia environment is properly initialized." + ) + + +def create_sysimage(sysimage_name="pysr.so", julia_project=None, quiet=False): + """Create a PackageCompiler.jl sysimage for SymbolicRegression.jl.""" Main = init_julia(julia_project=julia_project, quiet=quiet) cur_project_dir = Main.eval("dirname(Base.active_project())") sysimage_path = str(Path(cur_project_dir) / sysimage_name) diff --git a/pysr/sr.py b/pysr/sr.py index 760ec1c84..259d37ca2 100644 --- a/pysr/sr.py +++ b/pysr/sr.py @@ -21,13 +21,13 @@ check_is_fitted, ) +from .package_compiler import _update_julia_project from .julia_helpers import ( init_julia, _process_julia_project, is_julia_version_greater_eq, _escape_filename, _load_cluster_manager, - _update_julia_project, _load_backend, ) from .export_numpy import CallableEquation From 8616793febc4f8c707f6c3d5ac69fabe2b1ea72a Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Mon, 28 Nov 2022 12:38:56 -0500 Subject: [PATCH 08/19] Separate install and compile steps --- pysr/__init__.py | 3 +-- pysr/package_compiler.py | 10 +++------- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/pysr/__init__.py b/pysr/__init__.py index 2b7c64601..96365bf88 100644 --- a/pysr/__init__.py +++ b/pysr/__init__.py @@ -7,8 +7,7 @@ best_callable, best_row, ) -from .package_compiler import install +from .package_compiler import install, compile from .feynman_problems import Problem, FeynmanProblem from .export_jax import sympy2jax from .export_torch import sympy2torch -from .package_compiler import create_sysimage diff --git a/pysr/package_compiler.py b/pysr/package_compiler.py index 9c9acbd32..4123f8f31 100644 --- a/pysr/package_compiler.py +++ b/pysr/package_compiler.py @@ -50,9 +50,7 @@ def _update_julia_project(Main, is_shared, io_arg): raise ImportError(_import_error()) from e -def install( - julia_project=None, quiet=False, precompile=None, compile=False -): # pragma: no cover +def install(julia_project=None, quiet=False, precompile=None): # pragma: no cover """ Install PyCall.jl and all required dependencies for SymbolicRegression.jl. @@ -87,9 +85,6 @@ def install( if precompile: Main.eval(f"Pkg.precompile({io_arg})") - if compile: - create_sysimage(julia_project=julia_project, quiet=quiet) - if not quiet: warnings.warn( "It is recommended to restart Python after installing PySR's dependencies," @@ -97,11 +92,12 @@ def install( ) -def create_sysimage(sysimage_name="pysr.so", julia_project=None, quiet=False): +def compile(sysimage_name="pysr.so", julia_project=None, quiet=False): """Create a PackageCompiler.jl sysimage for SymbolicRegression.jl.""" Main = init_julia(julia_project=julia_project, quiet=quiet) cur_project_dir = Main.eval("dirname(Base.active_project())") sysimage_path = str(Path(cur_project_dir) / sysimage_name) from julia import PackageCompiler + from julia import SymbolicRegression PackageCompiler.create_sysimage(["SymbolicRegression"], sysimage_path=sysimage_path) From b2d538050caadda80bfd949bdc87b544057e46fe Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Mon, 28 Nov 2022 14:07:29 -0500 Subject: [PATCH 09/19] Working precompilation --- pysr/julia_helpers.py | 47 +++++++++++++++++++++++++++++++++++----- pysr/package_compiler.py | 9 ++++++-- pysr/test/test_env.py | 3 ++- 3 files changed, 51 insertions(+), 8 deletions(-) diff --git a/pysr/julia_helpers.py b/pysr/julia_helpers.py index b281f426f..76ee84e43 100644 --- a/pysr/julia_helpers.py +++ b/pysr/julia_helpers.py @@ -32,6 +32,27 @@ def _load_juliainfo(): return juliainfo +def _get_julia_project_dir(): + # Assumes it is in JULIA_PROJECT: + assert "JULIA_PROJECT" in os.environ and os.environ["JULIA_PROJECT"] != "" + try: + cmds = [ + "julia", + "-e using Pkg; print(Pkg.project().path)", + ] + julia_project_dir_str = subprocess.run( + cmds, + capture_output=True, + env=os.environ, + ).stdout.decode() + except FileNotFoundError: + env_path = os.environ["PATH"] + raise FileNotFoundError( + f"Julia is not installed in your PATH. Please install Julia and add it to your PATH.\n\nCurrent PATH: {env_path}", + ) + return Path(julia_project_dir_str).parent + + def _get_julia_env_dir(): # Have to manually get env dir: try: @@ -114,7 +135,7 @@ def _check_for_conflicting_libraries(): # pragma: no cover ) -def init_julia(julia_project=None, quiet=False, julia_kwargs=None, return_aux=False): +def init_julia(julia_project=None, quiet=False, sysimage_name=None, julia_kwargs=None, return_aux=False): """Initialize julia binary, turning off compiled modules if needed.""" global julia_initialized global julia_kwargs_at_initialization @@ -123,16 +144,32 @@ def init_julia(julia_project=None, quiet=False, julia_kwargs=None, return_aux=Fa if not julia_initialized: _check_for_conflicting_libraries() + _julia_version_assertion() + processed_julia_project, is_shared = _process_julia_project(julia_project) + _set_julia_project_env(processed_julia_project, is_shared) + + # TODO: Make checking optional. + # Check if sysimage exists: + if sysimage_name is None: + # TODO: Is there a faster way to get this dir? + expected_sysimage = _get_julia_project_dir() / "pysr.so" + print(f"Checking for sysimage at {expected_sysimage}") + # Check if this file exists: + if expected_sysimage.exists(): + sysimage_name = str(expected_sysimage) + if julia_kwargs is None: julia_kwargs = {"optimize": 3} - from julia.core import JuliaInfo, UnsupportedPythonError + if sysimage_name is not None and "sysimage" not in julia_kwargs: + sysimage = str(sysimage_name) + print(f"Found existing sysimage at {sysimage}. Loading.") + julia_kwargs["sysimage"] = sysimage_name - _julia_version_assertion() - processed_julia_project, is_shared = _process_julia_project(julia_project) - _set_julia_project_env(processed_julia_project, is_shared) + from julia.core import JuliaInfo, UnsupportedPythonError try: + # TODO: Can we just get env info from this? info = JuliaInfo.load(julia="julia") except FileNotFoundError: env_path = os.environ["PATH"] diff --git a/pysr/package_compiler.py b/pysr/package_compiler.py index 4123f8f31..f6ceb551c 100644 --- a/pysr/package_compiler.py +++ b/pysr/package_compiler.py @@ -92,12 +92,17 @@ def install(julia_project=None, quiet=False, precompile=None): # pragma: no cov ) -def compile(sysimage_name="pysr.so", julia_project=None, quiet=False): +def compile( + julia_project=None, + quiet=False, + sysimage_name="pysr.so", +): """Create a PackageCompiler.jl sysimage for SymbolicRegression.jl.""" Main = init_julia(julia_project=julia_project, quiet=quiet) cur_project_dir = Main.eval("dirname(Base.active_project())") sysimage_path = str(Path(cur_project_dir) / sysimage_name) from julia import PackageCompiler - from julia import SymbolicRegression + + Main.eval("using SymbolicRegression") PackageCompiler.create_sysimage(["SymbolicRegression"], sysimage_path=sysimage_path) diff --git a/pysr/test/test_env.py b/pysr/test/test_env.py index c044cfd0c..0d18412ad 100644 --- a/pysr/test/test_env.py +++ b/pysr/test/test_env.py @@ -5,6 +5,7 @@ from tempfile import TemporaryDirectory from .. import julia_helpers +from .. import package_compiler class TestJuliaProject(unittest.TestCase): @@ -29,7 +30,7 @@ def test_custom_shared_env(self): f'pushfirst!(DEPOT_PATH, "{julia_helpers._escape_filename(tmpdir)}")' ) test_env_name = "@pysr_test_env" - julia_helpers.install(julia_project=test_env_name) + package_compiler.install(julia_project=test_env_name) Main = julia_helpers.init_julia(julia_project=test_env_name) # Try to use env: From 65f77f77f4b2d8be979b15380419cf22f72ca372 Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Mon, 28 Nov 2022 14:23:01 -0500 Subject: [PATCH 10/19] Only try loading sysimage if Julia uninitialized --- pysr/julia_helpers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pysr/julia_helpers.py b/pysr/julia_helpers.py index 76ee84e43..fc2f8ad12 100644 --- a/pysr/julia_helpers.py +++ b/pysr/julia_helpers.py @@ -150,7 +150,7 @@ def init_julia(julia_project=None, quiet=False, sysimage_name=None, julia_kwargs # TODO: Make checking optional. # Check if sysimage exists: - if sysimage_name is None: + if sysimage_name is None and not julia_initialized: # TODO: Is there a faster way to get this dir? expected_sysimage = _get_julia_project_dir() / "pysr.so" print(f"Checking for sysimage at {expected_sysimage}") @@ -161,7 +161,7 @@ def init_julia(julia_project=None, quiet=False, sysimage_name=None, julia_kwargs if julia_kwargs is None: julia_kwargs = {"optimize": 3} - if sysimage_name is not None and "sysimage" not in julia_kwargs: + if sysimage_name is not None and "sysimage" not in julia_kwargs and not julia_initialized: sysimage = str(sysimage_name) print(f"Found existing sysimage at {sysimage}. Loading.") julia_kwargs["sysimage"] = sysimage_name From 2e2292685569bfd9ed76340821163cad5147a46e Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Mon, 28 Nov 2022 14:23:25 -0500 Subject: [PATCH 11/19] Clean up sysimage check --- pysr/julia_helpers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pysr/julia_helpers.py b/pysr/julia_helpers.py index fc2f8ad12..6d4c96280 100644 --- a/pysr/julia_helpers.py +++ b/pysr/julia_helpers.py @@ -153,7 +153,6 @@ def init_julia(julia_project=None, quiet=False, sysimage_name=None, julia_kwargs if sysimage_name is None and not julia_initialized: # TODO: Is there a faster way to get this dir? expected_sysimage = _get_julia_project_dir() / "pysr.so" - print(f"Checking for sysimage at {expected_sysimage}") # Check if this file exists: if expected_sysimage.exists(): sysimage_name = str(expected_sysimage) From a3be3c1155162549ab56515308a1d17cf261624e Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Tue, 29 Nov 2022 09:32:31 -0500 Subject: [PATCH 12/19] Ensure that we compile an optimized image --- pysr/package_compiler.py | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/pysr/package_compiler.py b/pysr/package_compiler.py index f6ceb551c..bea7fd03a 100644 --- a/pysr/package_compiler.py +++ b/pysr/package_compiler.py @@ -33,11 +33,18 @@ def _add_sr_to_julia_project(Main, io_arg): name="PackageCompiler", rev="v2.1.0", ) + Main.pycall_spec = Main.PackageSpec( + name="PyCall", + rev="v1.94.1", + ) Main.eval( "Pkg.add([" - + ", ".join(["sr_spec", "clustermanagers_spec", "packagecompiler_spec"]) + + ", ".join( + ["sr_spec", "clustermanagers_spec", "packagecompiler_spec", "pycall_spec"] + ) + f"], {io_arg})" ) + Main.eval(f'Pkg.build("PyCall", {io_arg})') def _update_julia_project(Main, is_shared, io_arg): @@ -98,11 +105,23 @@ def compile( sysimage_name="pysr.so", ): """Create a PackageCompiler.jl sysimage for SymbolicRegression.jl.""" - Main = init_julia(julia_project=julia_project, quiet=quiet) + Main = init_julia( + julia_project=julia_project, + quiet=quiet, + julia_kwargs={ + "compiled_modules": False, + "optimize": 3, + "threads": "auto", + "compile": "all", + }, + ) cur_project_dir = Main.eval("dirname(Base.active_project())") sysimage_path = str(Path(cur_project_dir) / sysimage_name) from julia import PackageCompiler + Main.eval("using PyCall") Main.eval("using SymbolicRegression") - PackageCompiler.create_sysimage(["SymbolicRegression"], sysimage_path=sysimage_path) + PackageCompiler.create_sysimage( + ["SymbolicRegression", "PyCall"], sysimage_path=sysimage_path + ) From f8c052264666db2f6dd7ff318e736b694d8109af Mon Sep 17 00:00:00 2001 From: "deepsource-autofix[bot]" <62050782+deepsource-autofix[bot]@users.noreply.github.com> Date: Mon, 28 Nov 2022 19:23:39 +0000 Subject: [PATCH 13/19] Format code with black --- pysr/julia_helpers.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pysr/julia_helpers.py b/pysr/julia_helpers.py index 6d4c96280..d14f84da7 100644 --- a/pysr/julia_helpers.py +++ b/pysr/julia_helpers.py @@ -160,7 +160,11 @@ def init_julia(julia_project=None, quiet=False, sysimage_name=None, julia_kwargs if julia_kwargs is None: julia_kwargs = {"optimize": 3} - if sysimage_name is not None and "sysimage" not in julia_kwargs and not julia_initialized: + if ( + sysimage_name is not None + and "sysimage" not in julia_kwargs + and not julia_initialized + ): sysimage = str(sysimage_name) print(f"Found existing sysimage at {sysimage}. Loading.") julia_kwargs["sysimage"] = sysimage_name From 3cb46f05629522598d28c06886e45b60bd59c03d Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Tue, 29 Nov 2022 10:04:08 -0500 Subject: [PATCH 14/19] Improve time to check project dir --- pysr/julia_helpers.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/pysr/julia_helpers.py b/pysr/julia_helpers.py index d14f84da7..bb6649eae 100644 --- a/pysr/julia_helpers.py +++ b/pysr/julia_helpers.py @@ -38,7 +38,11 @@ def _get_julia_project_dir(): try: cmds = [ "julia", - "-e using Pkg; print(Pkg.project().path)", + "--compile=min", + "--startup-file=no", + "-O0", + "-g0", + "-e import Pkg: project; print(project().path)", ] julia_project_dir_str = subprocess.run( cmds, @@ -56,8 +60,16 @@ def _get_julia_project_dir(): def _get_julia_env_dir(): # Have to manually get env dir: try: + cmds = [ + "julia", + "--compile=min", + "--startup-file=no", + "-O0", + "-g0", + "-e import Pkg: envdir; print(envdir())", + ] julia_env_dir_str = subprocess.run( - ["julia", "-e using Pkg; print(Pkg.envdir())"], + cmds, capture_output=True, env=os.environ, ).stdout.decode() From b438421c7d8af2cb492618658927d75244a13609 Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Tue, 29 Nov 2022 10:12:38 -0500 Subject: [PATCH 15/19] Ensure that compiling twice works --- pysr/package_compiler.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pysr/package_compiler.py b/pysr/package_compiler.py index bea7fd03a..938d5cb9e 100644 --- a/pysr/package_compiler.py +++ b/pysr/package_compiler.py @@ -3,6 +3,7 @@ import os from pathlib import Path import warnings +from multiprocessing import cpu_count import numpy as np from julia.api import JuliaError @@ -108,10 +109,11 @@ def compile( Main = init_julia( julia_project=julia_project, quiet=quiet, + use_sysimage=False, julia_kwargs={ "compiled_modules": False, "optimize": 3, - "threads": "auto", + "threads": cpu_count(), "compile": "all", }, ) From ded8eb04b4440d0d93d6ad06ad926d8d9a35d17c Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Tue, 29 Nov 2022 10:12:50 -0500 Subject: [PATCH 16/19] Allow forced init without sysimage --- pysr/julia_helpers.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/pysr/julia_helpers.py b/pysr/julia_helpers.py index bb6649eae..1dea3f6f3 100644 --- a/pysr/julia_helpers.py +++ b/pysr/julia_helpers.py @@ -147,7 +147,14 @@ def _check_for_conflicting_libraries(): # pragma: no cover ) -def init_julia(julia_project=None, quiet=False, sysimage_name=None, julia_kwargs=None, return_aux=False): +def init_julia( + julia_project=None, + quiet=False, + use_sysimage=True, + sysimage_name=None, + julia_kwargs=None, + return_aux=False, +): """Initialize julia binary, turning off compiled modules if needed.""" global julia_initialized global julia_kwargs_at_initialization @@ -162,7 +169,7 @@ def init_julia(julia_project=None, quiet=False, sysimage_name=None, julia_kwargs # TODO: Make checking optional. # Check if sysimage exists: - if sysimage_name is None and not julia_initialized: + if use_sysimage and sysimage_name is None and not julia_initialized: # TODO: Is there a faster way to get this dir? expected_sysimage = _get_julia_project_dir() / "pysr.so" # Check if this file exists: From 6dbda5c820a05543b502b11cedcb4c897ad11526 Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Tue, 29 Nov 2022 12:17:51 -0500 Subject: [PATCH 17/19] Fix re-install --- pysr/package_compiler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysr/package_compiler.py b/pysr/package_compiler.py index 938d5cb9e..fc97de6fc 100644 --- a/pysr/package_compiler.py +++ b/pysr/package_compiler.py @@ -75,7 +75,7 @@ def install(julia_project=None, quiet=False, precompile=None): # pragma: no cov os.environ["JULIA_PKG_PRECOMPILE_AUTO"] = "0" julia.install(quiet=quiet) - Main, init_log = init_julia(julia_project, quiet=quiet, return_aux=True) + Main, init_log = init_julia(julia_project, quiet=quiet, use_sysimage=False, return_aux=True) io_arg = _get_io_arg(quiet) if precompile is None: From f183a23d94d83f1b18fb07aba5175f66af1b541b Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Tue, 29 Nov 2022 12:18:04 -0500 Subject: [PATCH 18/19] Compile all packages into sysimage --- pysr/package_compiler.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pysr/package_compiler.py b/pysr/package_compiler.py index fc97de6fc..ce182575c 100644 --- a/pysr/package_compiler.py +++ b/pysr/package_compiler.py @@ -121,9 +121,6 @@ def compile( sysimage_path = str(Path(cur_project_dir) / sysimage_name) from julia import PackageCompiler - Main.eval("using PyCall") Main.eval("using SymbolicRegression") - PackageCompiler.create_sysimage( - ["SymbolicRegression", "PyCall"], sysimage_path=sysimage_path - ) + PackageCompiler.create_sysimage(sysimage_path=sysimage_path) From b3482873399dcd81b7b8eb3750236ead6a1ece61 Mon Sep 17 00:00:00 2001 From: "deepsource-autofix[bot]" <62050782+deepsource-autofix[bot]@users.noreply.github.com> Date: Thu, 23 Feb 2023 21:05:26 +0000 Subject: [PATCH 19/19] Format code with black --- pysr/package_compiler.py | 4 +++- pysr/sr.py | 2 -- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pysr/package_compiler.py b/pysr/package_compiler.py index ce182575c..59edb76c0 100644 --- a/pysr/package_compiler.py +++ b/pysr/package_compiler.py @@ -75,7 +75,9 @@ def install(julia_project=None, quiet=False, precompile=None): # pragma: no cov os.environ["JULIA_PKG_PRECOMPILE_AUTO"] = "0" julia.install(quiet=quiet) - Main, init_log = init_julia(julia_project, quiet=quiet, use_sysimage=False, return_aux=True) + Main, init_log = init_julia( + julia_project, quiet=quiet, use_sysimage=False, return_aux=True + ) io_arg = _get_io_arg(quiet) if precompile is None: diff --git a/pysr/sr.py b/pysr/sr.py index 259d37ca2..2fd6185eb 100644 --- a/pysr/sr.py +++ b/pysr/sr.py @@ -741,7 +741,6 @@ def __init__( julia_kwargs=None, **kwargs, ): - # Hyperparameters # - Model search parameters self.model_selection = model_selection @@ -2095,7 +2094,6 @@ def get_hof(self): equation_file_contents = copy.deepcopy(self.equation_file_contents_) for output in equation_file_contents: - scores = [] lastMSE = None lastComplexity = 0