From 0f84630beec75358c444ca08f9df711f33a53095 Mon Sep 17 00:00:00 2001 From: Pablo Alcaraz Date: Fri, 23 Dec 2016 14:30:34 -0800 Subject: [PATCH] Maven generates an auto installable jar with C libraries embedded. Only tested on Linux. When jpy is invoked from the JVM, C libraries are extracted to the temporal directory and jpy library is initialized. Therefore there is not need to install JPY by compiling jpy sources. It is enough to use the generated jar file. --- CHANGES.txt | 4 + pom.xml | 28 +++++++ setup.py | 3 +- src/main/java/org/jpy/DL.java | 18 +++-- src/main/java/org/jpy/PyLib.java | 129 +++++++++++++++++++++++++++++-- 5 files changed, 168 insertions(+), 14 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index ea578c4d7f..c24c8f4650 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -6,6 +6,10 @@ Version 0.9 ============= * Extended unit level tests +* Maven generates an auto installable jar with C libraries embedded. + Only tested on Linux. + When jpy is invoked from the JVM, C libraries are extracted to the temporal directory and jpy library is initialized. + Therefore there is not need to install JPY by compiling jpy sources. It is enough to use the generated jar file. Version 0.8.1 diff --git a/pom.xml b/pom.xml index 4a874803cb..751c27623e 100644 --- a/pom.xml +++ b/pom.xml @@ -113,6 +113,34 @@ + + org.apache.maven.plugins + maven-antrun-plugin + 1.8 + + + copy-binaries + compile + + + + + + + + + + + + + + run + + + + diff --git a/setup.py b/setup.py index 261545f204..67c571190a 100644 --- a/setup.py +++ b/setup.py @@ -34,6 +34,7 @@ from setuptools.extension import Extension from distutils import log +jpy_config = 'jpyconfig.properties' base_dir = os.path.dirname(os.path.abspath(__file__)) src_main_c_dir = os.path.join(base_dir, 'src', 'main', 'c') src_test_py_dir = os.path.join(base_dir, 'src', 'test', 'python') @@ -391,7 +392,7 @@ def restore(self): else: mvn_goal = 'test' mvn_args = '-DargLine=-Xmx512m -Djpy.config=' + os.path.join(build_dir, - 'jpyconfig.properties') + ' -Djpy.debug=true' + jpy_config) + ' -Djpy.debug=true' log.info("Executing Maven goal " + repr(mvn_goal) + " with arg line " + repr(mvn_args)) code = subprocess.call(['mvn', mvn_goal, mvn_args], shell=platform.system() == 'Windows') if code: diff --git a/src/main/java/org/jpy/DL.java b/src/main/java/org/jpy/DL.java index b9085b8eed..d1271de1ea 100644 --- a/src/main/java/org/jpy/DL.java +++ b/src/main/java/org/jpy/DL.java @@ -16,6 +16,8 @@ package org.jpy; +import java.io.File; + /** * A replacement for {@link System#load(String)} with support for POSIX {@code dlopen} flags. *

@@ -62,13 +64,15 @@ public class DL { public static native String dlerror(); static { - try { - System.loadLibrary("jdl"); - } catch (Throwable t) { - String jdlLibPath = System.getProperty("jpy.jdlLib"); - if (jdlLibPath != null) { - System.load(jdlLibPath); - } else { + String jdlLibPath = System.getProperty("jpy.jdlLib"); + if(jdlLibPath != null && new File(jdlLibPath).exists()) { + // load the library from the path. + System.load(jdlLibPath); + } else { + // try to load it using default libs. + try { + System.loadLibrary("jdl"); + } catch (Throwable t) { throw new RuntimeException("Failed to load 'jdl' shared library. You can use system property 'jpy.jdlLib' to specify it.", t); } } diff --git a/src/main/java/org/jpy/PyLib.java b/src/main/java/org/jpy/PyLib.java index f802fcbb90..2b56c51e8c 100644 --- a/src/main/java/org/jpy/PyLib.java +++ b/src/main/java/org/jpy/PyLib.java @@ -17,14 +17,18 @@ package org.jpy; import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.attribute.PosixFilePermissions; import java.util.ArrayList; import java.util.Map; +import java.util.Properties; -import static org.jpy.PyLibConfig.JPY_LIB_KEY; -import static org.jpy.PyLibConfig.OS; -import static org.jpy.PyLibConfig.PYTHON_LIB_KEY; -import static org.jpy.PyLibConfig.getOS; -import static org.jpy.PyLibConfig.getProperty; +import static org.jpy.PyLibConfig.*; /** * Represents the library that provides the Python interpreter (CPython). @@ -55,6 +59,10 @@ public class PyLib { private static String dllFilePath; private static Throwable dllProblem; private static boolean dllLoaded; + private static final String JPY_SO_FILE = "jpy.so"; + private static final String JDL_SO_FILE = "jdl.so"; + private static final String JPY_CONFIG_TEMPLATE = "jpyconfig.properties.template"; + private static final String JPY_CONFIG_KEY = "jpy.config"; /** * The kind of callable Python objects. @@ -378,7 +386,8 @@ private static void preloadPythonLib(String pythonLibPath) { } else { // Fixes https://github.com/bcdev/jpy/issues/58 // Loading of jpy DLL fails for user-specific Python installations on Windows - if (DEBUG) System.out.printf("org.jpy.PyLib: System.load(\"%s\")%n", pythonLibPath); + if (DEBUG) + System.out.printf("org.jpy.PyLib: System.load(\"%s\")%n", pythonLibPath); try { System.load(pythonLibPath); } catch (Exception e) { @@ -391,11 +400,119 @@ private static void preloadPythonLib(String pythonLibPath) { private PyLib() { } + private static int randomInt = (int) (Math.random() * 1000000); + static { if (DEBUG) System.out.println("org.jpy.PyLib: entered static initializer"); + tryAutoConfiguration(); loadLib(); if (DEBUG) System.out.println("org.jpy.PyLib: exited static initializer"); } + + private static void tryAutoConfiguration() { + if (DEBUG) System.out.println("org.jpy.PyLibConfig: trying to use C libraries from jar file."); + + File jpySoFile = extractToTempFile(JPY_SO_FILE); + if (DEBUG) System.out.println("org.jpy.PyLibConfig: jpy.so file: " + jpySoFile.getAbsolutePath()); + File jdlSoFile = extractToTempFile(JDL_SO_FILE); + if (DEBUG) System.out.println("org.jpy.PyLibConfig: jdl.so file: " + jdlSoFile.getAbsolutePath()); + if(jpySoFile != null && jdlSoFile != null) { + File jpyConfigFile = createTempConfigFile(jpySoFile, jdlSoFile); + if (DEBUG) System.out.println("org.jpy.PyLibConfig: Setting property " + + JPY_CONFIG_KEY + " to " + jpyConfigFile.getAbsolutePath() + "."); + System.setProperty(JPY_CONFIG_KEY, jpyConfigFile.getAbsolutePath()); + } + + + if (DEBUG) System.out.println("org.jpy.PyLibConfig: End auto configuration."); + + } + + private static File extractToTempFile(String internalFilename) { + File filename = null; + InputStream fileStream = Thread.currentThread().getContextClassLoader().getResourceAsStream(internalFilename); + if (fileStream != null) { + OutputStream output = null; + byte[] buffer = new byte[8 * 1024]; + try { + filename = new File(System.getProperty("java.io.tmpdir"), "jpy-" + randomInt); + filename.mkdirs(); + filename = new File(filename, internalFilename); + filename.createNewFile(); + Files.setAttribute(filename.toPath(), "posix:permissions", PosixFilePermissions.fromString("rwxrwxrwx")); + filename.deleteOnExit(); + output = new FileOutputStream(filename); + int bytesRead; + while ((bytesRead = fileStream.read(buffer)) != -1) { + output.write(buffer, 0, bytesRead); + } + } catch (FileNotFoundException e) { + System.out.println("org.jpy.PyLibConfig: Error extracting file " + + internalFilename + " from jpy.jar file."); + e.printStackTrace(); + } catch (IOException e) { + System.out.println("org.jpy.PyLibConfig: Error extracting file " + + internalFilename + " from jpy.jar file."); + e.printStackTrace(); + } finally { + try { + fileStream.close(); + if (output != null) { + output.close(); + } + } catch (IOException e) { + if (DEBUG) System.out.println("org.jpy.PyLibConfig: File " + internalFilename + + " not found inside jpy.jar library. Proceeding as usual."); + } + } + } + return filename; + } + + private static File createTempConfigFile(File jpySoFile, File jdlSoFile) { + File filename = null; + OutputStream output = null; + try { + InputStream templateStream = Thread.currentThread().getContextClassLoader().getResourceAsStream(JPY_CONFIG_TEMPLATE); + if (templateStream != null) { + Properties p = new Properties(); + p.load(templateStream); + filename = File.createTempFile("jpy-", ".jpyconfig.properties"); + Files.setAttribute(filename.toPath(), "posix:permissions", PosixFilePermissions.fromString("rwxrwxrwx")); + filename.deleteOnExit(); + output = new FileOutputStream(filename); + String comment = "# Created by 'PyLib' launcher on 2016-12-07 17:39:38.232756\n" + + "# This file is read by the jpy Java API (org.jpy.PyLib class) in order to find shared libraries\n"; +// "jpy.jpyLib = " + jpySoFile.getAbsolutePath() + "\n" + +// "jpy.jdlLib = " + jdlSoFile.getAbsolutePath() + "\n" + +// "jpy.pythonLib = /usr/lib/x86_64-linux-gnu/libpython2.7.so\n" + +// "jpy.pythonPrefix = /usr\n" + +// "jpy.pythonExecutable = /usr/bin/python"; + Properties temporalProperties = new Properties(); + temporalProperties.setProperty("jpy.jpyLib", jpySoFile.getAbsolutePath()); + temporalProperties.setProperty("jpy.jdlLib", jdlSoFile.getAbsolutePath()); + temporalProperties.setProperty("jpy.pythonLib", p.getProperty("jpy.pythonLib")); + temporalProperties.setProperty("jpy.pythonPrefix", p.getProperty("jpy.pythonPrefix")); + temporalProperties.setProperty("jpy.pythonExecutable", p.getProperty("jpy.pythonExecutable")); + temporalProperties.store(output, comment); + } + } catch (IOException e) { + System.out.println("org.jpy.PyLibConfig: Error reading template file " + + JPY_CONFIG_TEMPLATE + " from jpy.jar file."); + e.printStackTrace(); + } finally { + if(output != null) { + try { + output.close(); + } catch (IOException e) { + if (DEBUG) System.out.println("org.jpy.PyLibConfig: File " + JPY_CONFIG_TEMPLATE + + " not found inside jpy.jar library. Proceeding as usual."); + } + } + } + return filename; + } + }