From 84f9cfb5a71ce296e3653002885a7aa521613cde Mon Sep 17 00:00:00 2001 From: Kushal Pisavadia Date: Tue, 18 Mar 2025 17:49:41 +0000 Subject: [PATCH 01/14] C library for Pkl MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This introduces native C bindings for Pkl. Dynamic Library --------------- Using `org.graalvm.nativeimage` and the `native-image` binary we produce a dynamic library (for each OS/Arch variant) that provides a way to initialise itself with a `graal_isolatethread_t`. Methods are annotated with `@CEntryPoint` and exported. This change results in an architecture and OS-specific directory being created which now produces the headers for our shared library functions: ``` ❯ ll libpkl/build/libs/macos-aarch64/ graal_isolate_dynamic.h graal_isolate.h libpkl-internal_dynamic.h libpkl-internal-macos-aarch64_dynamic.h libpkl-internal-macos-aarch64.dylib libpkl-internal-macos-aarch64.h libpkl-internal.dylib libpkl-internal.h ``` `libpkl` -------- The produced `libpkl` dynamic library wraps the GraalVM C interface into something that is future-friendly for the needs of a Pkl integrator. It exports an interface which aligns with SPICE-0015[1]. ``` ❯ ll libpkl/build/libs/macos-aarch64/ graal_isolate_dynamic.h graal_isolate.h libpkl-internal_dynamic.h libpkl-internal-macos-aarch64_dynamic.h libpkl-internal-macos-aarch64.dylib libpkl-internal-macos-aarch64.h libpkl-internal.dylib libpkl-internal.h libpkl.dylib <--- this is new libpkl.h <--- this is new ``` JNA --- Testing of the produced `libpkl` dynamic library is done using Java Native Access[2] for ease. We provide an `interface` in Kotlin which JNA transposes against the `libpkl` dynamic library discoverable at the path that is discoverable with `jna.library.path`. Load in `projects.pklCommonsCli` to deal with `UnsupportedFeatureException` --------------------------------------------------------------------------- This is to deal with the following error: ``` Caused by: com.oracle.graal.pointsto.constraints.UnsupportedFeatureException: Detected a started Thread in the image heap. Thread name: main. Threads running in the image generator are no longer running at image runtime. If these objects should not be stored in the image heap, you can use '--trace-object-instantiation=java.lang.Thread' ``` [1] https://github.com/apple/pkl-evolution/pull/16 --- buildSrc/src/main/kotlin/NativeImageBuild.kt | 6 +- libpkl/gradle.lockfile | 81 ++ libpkl/libpkl.gradle.kts | 319 +++++++ libpkl/src/main/c/libpkl.c | 70 ++ libpkl/src/main/c/libpkl.h | 7 + .../src/main/java/org/pkl/libpkl/LibPkl.java | 89 ++ .../java/org/pkl/libpkl/LibPklLogger.java | 29 + .../org/pkl/libpkl/NativeInputStream.java | 43 + .../java/org/pkl/libpkl/NativeTransport.java | 62 ++ .../kotlin/org/pkl/libpkl/ILibPklLibrary.kt | 37 + .../src/test/kotlin/org/pkl/libpkl/JNATest.kt | 866 ++++++++++++++++++ .../kotlin/org/pkl/libpkl/JNATestClient.kt | 116 +++ .../pkl/libpkl/MessagePackDebugRenderer.kt | 107 +++ .../pkl/core/messaging/MessageTransports.java | 4 +- settings.gradle.kts | 2 + 15 files changed, 1835 insertions(+), 3 deletions(-) create mode 100644 libpkl/gradle.lockfile create mode 100644 libpkl/libpkl.gradle.kts create mode 100644 libpkl/src/main/c/libpkl.c create mode 100644 libpkl/src/main/c/libpkl.h create mode 100644 libpkl/src/main/java/org/pkl/libpkl/LibPkl.java create mode 100644 libpkl/src/main/java/org/pkl/libpkl/LibPklLogger.java create mode 100644 libpkl/src/main/java/org/pkl/libpkl/NativeInputStream.java create mode 100644 libpkl/src/main/java/org/pkl/libpkl/NativeTransport.java create mode 100644 libpkl/src/test/kotlin/org/pkl/libpkl/ILibPklLibrary.kt create mode 100644 libpkl/src/test/kotlin/org/pkl/libpkl/JNATest.kt create mode 100644 libpkl/src/test/kotlin/org/pkl/libpkl/JNATestClient.kt create mode 100644 libpkl/src/test/kotlin/org/pkl/libpkl/MessagePackDebugRenderer.kt diff --git a/buildSrc/src/main/kotlin/NativeImageBuild.kt b/buildSrc/src/main/kotlin/NativeImageBuild.kt index a16ea723b..727640ffa 100644 --- a/buildSrc/src/main/kotlin/NativeImageBuild.kt +++ b/buildSrc/src/main/kotlin/NativeImageBuild.kt @@ -16,6 +16,7 @@ import javax.inject.Inject import org.gradle.api.DefaultTask import org.gradle.api.file.ConfigurableFileCollection +import org.gradle.api.file.DirectoryProperty import org.gradle.api.provider.ListProperty import org.gradle.api.provider.Property import org.gradle.api.provider.Provider @@ -24,6 +25,7 @@ import org.gradle.api.services.BuildServiceParameters import org.gradle.api.tasks.ClasspathNormalizer import org.gradle.api.tasks.Input import org.gradle.api.tasks.InputFiles +import org.gradle.api.tasks.OutputDirectory import org.gradle.api.tasks.OutputFile import org.gradle.api.tasks.PathSensitivity import org.gradle.api.tasks.TaskAction @@ -49,7 +51,7 @@ abstract class NativeImageBuild : DefaultTask() { @get:InputFiles abstract val classpath: ConfigurableFileCollection - private val outputDir = project.layout.buildDirectory.dir("executable") + @get:OutputDirectory abstract val outputDir: DirectoryProperty @get:OutputFile val outputFile = outputDir.flatMap { it.file(imageName) } @@ -89,6 +91,8 @@ abstract class NativeImageBuild : DefaultTask() { // CPU resources). usesService(buildService) + outputDir.convention(project.layout.buildDirectory.dir("executable")) + group = "build" inputs diff --git a/libpkl/gradle.lockfile b/libpkl/gradle.lockfile new file mode 100644 index 000000000..90ef20152 --- /dev/null +++ b/libpkl/gradle.lockfile @@ -0,0 +1,81 @@ +# This is a Gradle generated file for dependency locking. +# Manual edits can break the build and are not advised. +# This file is expected to be part of source control. +com.github.ajalt.clikt:clikt-core-jvm:5.0.3=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +com.github.ajalt.clikt:clikt-core:5.0.3=apiDependenciesMetadata,compileClasspath,implementationDependenciesMetadata,runtimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath +com.github.ajalt.clikt:clikt-jvm:5.0.3=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +com.github.ajalt.clikt:clikt-markdown-jvm:5.0.3=runtimeClasspath,testRuntimeClasspath +com.github.ajalt.clikt:clikt-markdown:5.0.3=runtimeClasspath,testRuntimeClasspath +com.github.ajalt.clikt:clikt:5.0.3=apiDependenciesMetadata,compileClasspath,implementationDependenciesMetadata,runtimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath +com.github.ajalt.colormath:colormath-jvm:3.6.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +com.github.ajalt.colormath:colormath:3.6.0=apiDependenciesMetadata,compileClasspath,implementationDependenciesMetadata,runtimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath +com.github.ajalt.mordant:mordant-core-jvm:3.0.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +com.github.ajalt.mordant:mordant-core:3.0.1=apiDependenciesMetadata,compileClasspath,implementationDependenciesMetadata,runtimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath +com.github.ajalt.mordant:mordant-jvm-ffm-jvm:3.0.1=runtimeClasspath,testRuntimeClasspath +com.github.ajalt.mordant:mordant-jvm-ffm:3.0.1=runtimeClasspath,testRuntimeClasspath +com.github.ajalt.mordant:mordant-jvm-graal-ffi-jvm:3.0.1=runtimeClasspath,testRuntimeClasspath +com.github.ajalt.mordant:mordant-jvm-graal-ffi:3.0.1=runtimeClasspath,testRuntimeClasspath +com.github.ajalt.mordant:mordant-jvm-jna-jvm:3.0.1=runtimeClasspath,testRuntimeClasspath +com.github.ajalt.mordant:mordant-jvm-jna:3.0.1=runtimeClasspath,testRuntimeClasspath +com.github.ajalt.mordant:mordant-jvm:3.0.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +com.github.ajalt.mordant:mordant-markdown-jvm:3.0.1=runtimeClasspath,testRuntimeClasspath +com.github.ajalt.mordant:mordant-markdown:3.0.1=runtimeClasspath,testRuntimeClasspath +com.github.ajalt.mordant:mordant:3.0.1=apiDependenciesMetadata,compileClasspath,implementationDependenciesMetadata,runtimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath +net.bytebuddy:byte-buddy:1.15.11=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath +net.java.dev.jna:jna-platform:5.17.0=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath +net.java.dev.jna:jna:5.14.0=runtimeClasspath +net.java.dev.jna:jna:5.17.0=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath +org.apiguardian:apiguardian-api:1.1.2=testCompileClasspath,testImplementationDependenciesMetadata,testJdk17CompileClasspath,testJdk17ImplementationDependenciesMetadata +org.assertj:assertj-core:3.27.3=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath +org.graalvm.polyglot:polyglot:24.1.2=compileClasspath,compileOnlyDependenciesMetadata,implementationDependenciesMetadata,runtimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath +org.graalvm.sdk:collections:24.1.2=compileClasspath,compileOnlyDependenciesMetadata,implementationDependenciesMetadata,runtimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath +org.graalvm.sdk:graal-sdk:24.1.2=compileClasspath,compileOnlyDependenciesMetadata,runtimeClasspath,testRuntimeClasspath +org.graalvm.sdk:jniutils:24.1.2=compileClasspath,implementationDependenciesMetadata,runtimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath +org.graalvm.sdk:nativeimage:24.1.2=compileClasspath,compileOnlyDependenciesMetadata,implementationDependenciesMetadata,runtimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath +org.graalvm.sdk:word:24.1.2=compileClasspath,compileOnlyDependenciesMetadata,implementationDependenciesMetadata,runtimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath +org.graalvm.truffle:truffle-api:24.1.2=compileClasspath,implementationDependenciesMetadata,runtimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath +org.graalvm.truffle:truffle-compiler:24.1.2=compileClasspath,implementationDependenciesMetadata,runtimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath +org.graalvm.truffle:truffle-runtime:24.1.2=compileClasspath,implementationDependenciesMetadata,runtimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath +org.jetbrains.intellij.deps:trove4j:1.0.20200330=kotlinBuildToolsApiClasspath,kotlinCompilerClasspath,kotlinKlibCommonizerClasspath +org.jetbrains.kotlin:kotlin-build-common:2.0.21=kotlinBuildToolsApiClasspath +org.jetbrains.kotlin:kotlin-build-tools-api:2.0.21=kotlinBuildToolsApiClasspath +org.jetbrains.kotlin:kotlin-build-tools-impl:2.0.21=kotlinBuildToolsApiClasspath +org.jetbrains.kotlin:kotlin-compiler-embeddable:2.0.21=kotlinBuildToolsApiClasspath,kotlinCompilerClasspath,kotlinKlibCommonizerClasspath +org.jetbrains.kotlin:kotlin-compiler-runner:2.0.21=kotlinBuildToolsApiClasspath +org.jetbrains.kotlin:kotlin-daemon-client:2.0.21=kotlinBuildToolsApiClasspath +org.jetbrains.kotlin:kotlin-daemon-embeddable:2.0.21=kotlinBuildToolsApiClasspath,kotlinCompilerClasspath,kotlinKlibCommonizerClasspath +org.jetbrains.kotlin:kotlin-klib-commonizer-embeddable:2.0.21=kotlinKlibCommonizerClasspath +org.jetbrains.kotlin:kotlin-native-prebuilt:2.0.21=kotlinNativeBundleConfiguration +org.jetbrains.kotlin:kotlin-reflect:2.0.21=kotlinBuildToolsApiClasspath,kotlinCompilerClasspath,kotlinKlibCommonizerClasspath +org.jetbrains.kotlin:kotlin-script-runtime:2.0.21=kotlinBuildToolsApiClasspath,kotlinCompilerClasspath,kotlinCompilerPluginClasspathMain,kotlinCompilerPluginClasspathTest,kotlinCompilerPluginClasspathTestJdk17,kotlinKlibCommonizerClasspath +org.jetbrains.kotlin:kotlin-scripting-common:2.0.21=kotlinBuildToolsApiClasspath,kotlinCompilerPluginClasspathMain,kotlinCompilerPluginClasspathTest,kotlinCompilerPluginClasspathTestJdk17 +org.jetbrains.kotlin:kotlin-scripting-compiler-embeddable:2.0.21=kotlinBuildToolsApiClasspath,kotlinCompilerPluginClasspathMain,kotlinCompilerPluginClasspathTest,kotlinCompilerPluginClasspathTestJdk17 +org.jetbrains.kotlin:kotlin-scripting-compiler-impl-embeddable:2.0.21=kotlinBuildToolsApiClasspath,kotlinCompilerPluginClasspathMain,kotlinCompilerPluginClasspathTest,kotlinCompilerPluginClasspathTestJdk17 +org.jetbrains.kotlin:kotlin-scripting-jvm:2.0.21=kotlinBuildToolsApiClasspath,kotlinCompilerPluginClasspathMain,kotlinCompilerPluginClasspathTest,kotlinCompilerPluginClasspathTestJdk17 +org.jetbrains.kotlin:kotlin-stdlib-jdk7:2.0.21=apiDependenciesMetadata,compileClasspath,implementationDependenciesMetadata,runtimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath +org.jetbrains.kotlin:kotlin-stdlib-jdk8:2.0.21=apiDependenciesMetadata,compileClasspath,implementationDependenciesMetadata,runtimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath +org.jetbrains.kotlin:kotlin-stdlib:2.0.21=apiDependenciesMetadata,compileClasspath,implementationDependenciesMetadata,kotlinBuildToolsApiClasspath,kotlinCompilerClasspath,kotlinCompilerPluginClasspathMain,kotlinCompilerPluginClasspathTest,kotlinCompilerPluginClasspathTestJdk17,kotlinKlibCommonizerClasspath,runtimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath +org.jetbrains.kotlinx:kotlinx-coroutines-core-jvm:1.6.4=kotlinBuildToolsApiClasspath,kotlinCompilerClasspath,kotlinKlibCommonizerClasspath +org.jetbrains:annotations:13.0=compileClasspath,kotlinBuildToolsApiClasspath,kotlinCompilerClasspath,kotlinCompilerPluginClasspathMain,kotlinCompilerPluginClasspathTest,kotlinCompilerPluginClasspathTestJdk17,kotlinKlibCommonizerClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.jetbrains:markdown-jvm:0.7.3=runtimeClasspath,testRuntimeClasspath +org.jetbrains:markdown:0.7.3=runtimeClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter-api:5.11.4=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath +org.junit.jupiter:junit-jupiter-api:5.8.2=testJdk17CompileClasspath,testJdk17ImplementationDependenciesMetadata,testJdk17RuntimeClasspath +org.junit.jupiter:junit-jupiter-engine:5.11.4=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath +org.junit.jupiter:junit-jupiter-engine:5.8.2=testJdk17RuntimeClasspath +org.junit.jupiter:junit-jupiter-params:5.11.4=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath +org.junit.jupiter:junit-jupiter-params:5.8.2=testJdk17CompileClasspath,testJdk17ImplementationDependenciesMetadata,testJdk17RuntimeClasspath +org.junit.jupiter:junit-jupiter:5.8.2=testJdk17CompileClasspath,testJdk17ImplementationDependenciesMetadata,testJdk17RuntimeClasspath +org.junit.platform:junit-platform-commons:1.12.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath +org.junit.platform:junit-platform-commons:1.8.2=testJdk17CompileClasspath,testJdk17ImplementationDependenciesMetadata,testJdk17RuntimeClasspath +org.junit.platform:junit-platform-engine:1.12.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath +org.junit.platform:junit-platform-engine:1.8.2=testJdk17RuntimeClasspath +org.junit.platform:junit-platform-launcher:1.8.2=testJdk17RuntimeClasspath +org.junit:junit-bom:5.11.4=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath +org.junit:junit-bom:5.8.2=testJdk17CompileClasspath,testJdk17ImplementationDependenciesMetadata,testJdk17RuntimeClasspath +org.msgpack:msgpack-core:0.9.8=compileClasspath,implementationDependenciesMetadata,runtimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath +org.opentest4j:opentest4j:1.2.0=testJdk17CompileClasspath,testJdk17ImplementationDependenciesMetadata,testJdk17RuntimeClasspath +org.opentest4j:opentest4j:1.3.0=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath +org.organicdesign:Paguro:3.10.3=runtimeClasspath,testRuntimeClasspath +org.snakeyaml:snakeyaml-engine:2.9=runtimeClasspath,testRuntimeClasspath +empty=annotationProcessor,intransitiveDependenciesMetadata,kotlinCompilerPluginClasspath,kotlinNativeCompilerPluginClasspath,kotlinScriptDefExtensions,shadow,signatures,sourcesJar,stagedAlpineLinuxAmd64NativeLibrary,stagedLinuxAarch64NativeLibrary,stagedLinuxAmd64NativeLibrary,stagedMacAarch64NativeLibrary,stagedMacAmd64NativeLibrary,stagedWindowsAmd64NativeLibrary,testAnnotationProcessor,testApiDependenciesMetadata,testCompileOnlyDependenciesMetadata,testIntransitiveDependenciesMetadata,testJdk17AnnotationProcessor,testJdk17ApiDependenciesMetadata,testJdk17CompileOnlyDependenciesMetadata,testJdk17IntransitiveDependenciesMetadata,testJdk17KotlinScriptDefExtensions,testKotlinScriptDefExtensions diff --git a/libpkl/libpkl.gradle.kts b/libpkl/libpkl.gradle.kts new file mode 100644 index 000000000..37fcbaba5 --- /dev/null +++ b/libpkl/libpkl.gradle.kts @@ -0,0 +1,319 @@ +/* + * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +plugins { + pklAllProjects + pklGraalVm + pklJavaLibrary + pklJavaExecutable + pklNativeLifecycle +} + +// assumes that `pklJavaExecutable` is also applied +val executableSpec = project.extensions.getByType() + +val stagedMacAmd64NativeLibrary: Configuration by configurations.creating +val stagedMacAarch64NativeLibrary: Configuration by configurations.creating +val stagedLinuxAmd64NativeLibrary: Configuration by configurations.creating +val stagedLinuxAarch64NativeLibrary: Configuration by configurations.creating +val stagedAlpineLinuxAmd64NativeLibrary: Configuration by configurations.creating +val stagedWindowsAmd64NativeLibrary: Configuration by configurations.creating + +dependencies { + compileOnly(libs.graalSdk) + + implementation(projects.pklCore) + implementation(projects.pklServer) + + api(projects.pklCommonsCli) + + implementation(libs.msgpack) + implementation(libs.truffleApi) + implementation(libs.truffleRuntime) + + testImplementation(projects.pklCommonsTest) + testImplementation("net.java.dev.jna:jna:5.17.0") + testImplementation("net.java.dev.jna:jna-platform:5.17.0") + + fun sharedLibrary(osAndArch: String) = files(nativeLibraryOutputFiles(osAndArch)) + + stagedMacAarch64NativeLibrary(sharedLibrary("macos-aarch64")) + stagedMacAmd64NativeLibrary(sharedLibrary("macos-amd64")) + stagedLinuxAmd64NativeLibrary(sharedLibrary("linux-amd64")) + stagedLinuxAarch64NativeLibrary(sharedLibrary("linux-aarch64")) + stagedAlpineLinuxAmd64NativeLibrary(sharedLibrary("alpine-linux-amd64")) + stagedWindowsAmd64NativeLibrary(sharedLibrary("windows-amd64.exe")) +} + +executable { + name = "libpkl-internal" + + // TODO(kushal): Why is all of this necessary now? Can it be stripped back? + javaName = "libpkl" + documentationName = "Pkl Native Library" + publicationName = "libpkl" + javaPublicationName = "libpkl" + mainClass = "org.pkl.libpkl.LibPkl" + website = "TODO" +} + +private fun extension(osAndArch: String) = + when (osAndArch.split("-").dropWhile { it == "alpine" }.first()) { + "linux" -> "so" + "macos" -> "dylib" + "unix" -> "so" + "windows" -> "dll" + else -> { + throw StopExecutionException( + "Don't know how to construct library extension for OS: ${osAndArch.split("-").first()}" + ) + } + } + +private fun nativeLibraryOutputFiles(osAndArch: String) = + project.layout.buildDirectory.dir("libs/$osAndArch").map { outputDir -> + // TODO(kushal): dashes/underscores for library files? C convention assumes underscores. + val libraryName = "libpkl-$osAndArch" + val libraryOutputFiles = + listOf( + "${libraryName}.${extension(osAndArch)}", + "${libraryName}_dynamic.h", + "${libraryName}.h", + + // GraalVM shared headers. + "graal_isolate.h", + "graal_isolate_dynamic.h", + ) + + libraryOutputFiles.map { filename -> outputDir.file(filename) } + } + +private fun NativeImageBuild.setOutputFiles(osAndArch: String) { + outputs.files(nativeLibraryOutputFiles(osAndArch)) +} + +private fun NativeImageBuild.amd64() { + arch = Architecture.AMD64 + dependsOn(":installGraalVmAmd64") +} + +private fun NativeImageBuild.aarch64() { + arch = Architecture.AARCH64 + dependsOn(":installGraalVmAarch64") +} + +private fun NativeImageBuild.setClasspath() { + classpath.from(sourceSets.main.map { it.output }) + classpath.from( + project(":pkl-commons-cli").extensions.getByType(SourceSetContainer::class)["svm"].output + ) + classpath.from(configurations.runtimeClasspath) +} + +val macNativeLibraryAmd64 by + tasks.registering(NativeImageBuild::class) { + outputDir = project.layout.buildDirectory.dir("libs/macos-amd64") + imageName = executableSpec.name.map { "$it-macos-amd64" } + mainClass = executableSpec.mainClass + amd64() + setClasspath() + extraNativeImageArgs = listOf("--shared") + + setOutputFiles("macos-amd64") + } + +val macNativeLibraryAarch64 by + tasks.registering(NativeImageBuild::class) { + outputDir = project.layout.buildDirectory.dir("libs/macos-aarch64") + imageName = executableSpec.name.map { "$it-macos-aarch64" } + mainClass = executableSpec.mainClass + aarch64() + setClasspath() + extraNativeImageArgs = listOf("--shared") + + setOutputFiles("macos-aarch64") + } + +val linuxNativeLibraryAmd64 by + tasks.registering(NativeImageBuild::class) { + outputDir = project.layout.buildDirectory.dir("libs/linux-amd64") + imageName = executableSpec.name.map { "$it-linux-amd64" } + mainClass = executableSpec.mainClass + amd64() + setClasspath() + extraNativeImageArgs = listOf("--shared") + + setOutputFiles("linux-amd64") + } + +val linuxNativeLibraryAarch64 by + tasks.registering(NativeImageBuild::class) { + outputDir = project.layout.buildDirectory.dir("libs/linux-aarch64") + imageName = executableSpec.name.map { "$it-linux-aarch64" } + mainClass = executableSpec.mainClass + aarch64() + setClasspath() + + extraNativeImageArgs = + listOf( + "--shared", + // Ensure compatibility for kernels with page size set to 4k, 16k and 64k + // (e.g. Raspberry Pi 5, Asahi Linux) + "-H:PageSize=65536", + ) + + setOutputFiles("linux-aarch64") + } + +val alpineNativeLibraryAmd64 by + tasks.registering(NativeImageBuild::class) { + outputDir = project.layout.buildDirectory.dir("libs/alpine-linux-amd64") + imageName = executableSpec.name.map { "$it-alpine-linux-amd64" } + mainClass = executableSpec.mainClass + amd64() + setClasspath() + + extraNativeImageArgs = + listOf( + "--shared", + // TODO(kushal): https://github.com/oracle/graal/issues/3053 + "--libc=musl", + ) + + setOutputFiles("alpine-linux-amd64") + } + +val windowsNativeLibraryAmd64 by + tasks.registering(NativeImageBuild::class) { + outputDir = project.layout.buildDirectory.dir("libs/windows-amd64") + imageName = executableSpec.name.map { "$it-windows-amd64" } + mainClass = executableSpec.mainClass + amd64() + setClasspath() + extraNativeImageArgs = listOf("--shared", "-Dfile.encoding=UTF-8") + + setOutputFiles("windows-amd64") + } + +val assembleNative by + tasks.existing { + // TODO(kushal): Remove this later. Only exists to debug output files are in the graph. + finalizedBy(validateNativeLibraryFilestasks) + } + +// TODO(kushal): Remove this later. Only exists to debug output files are in the graph. +val validateNativeLibraryFilestasks by + tasks.registering { + val assembleTasks = mutableSetOf>() + + when { + buildInfo.os.isMacOsX -> { + assembleTasks.add(macNativeLibraryAmd64) + if (buildInfo.arch == "aarch64") { + assembleTasks.add(macNativeLibraryAarch64) + } + } + + buildInfo.os.isWindows -> { + assembleTasks.add(windowsNativeLibraryAmd64) + } + + buildInfo.os.isLinux && buildInfo.arch == "aarch64" -> { + assembleTasks.add(linuxNativeLibraryAarch64) + } + + buildInfo.os.isLinux && buildInfo.arch == "amd64" -> { + assembleTasks.add(linuxNativeLibraryAmd64) + if (buildInfo.hasMuslToolchain) { + assembleTasks.add(alpineNativeLibraryAmd64) + } + } + } + + dependsOn(assembleTasks) + + doLast { + for (taskProvider in assembleTasks) { + val task = taskProvider.get() + val outputFiles = task.outputs.files.files + + println("==== Validating Native Library Files Exist ====") + println("${task.name} outputs:") + outputFiles.forEach { file -> println("- ${file.absolutePath} (exists: ${file.exists()})") } + } + } + } + +// Expose underlying task's outputs +private fun Task.wraps(other: TaskProvider) { + dependsOn(other) + outputs.files(other) +} + +val assembleNativeMacOsAarch64 by tasks.existing { wraps(macNativeLibraryAarch64) } + +val assembleNativeMacOsAmd64 by tasks.existing { wraps(macNativeLibraryAmd64) } + +val assembleNativeLinuxAarch64 by tasks.existing { wraps(linuxNativeLibraryAarch64) } + +val assembleNativeLinuxAmd64 by tasks.existing { wraps(linuxNativeLibraryAmd64) } + +val assembleNativeAlpineLinuxAmd64 by tasks.existing { wraps(alpineNativeLibraryAmd64) } + +val assembleNativeWindowsAmd64 by tasks.existing { wraps(windowsNativeLibraryAmd64) } + +val macNativeFullLibraryAarch64 by + tasks.registering(Exec::class) { + dependsOn(macNativeLibraryAarch64) + + val libraryOutputDir = project.layout.buildDirectory.dir("libs/macos-aarch64").get() + val projectDir = project.layout.projectDirectory.asFile.path + + workingDir = libraryOutputDir.asFile + + // TODO: Make this portable. + commandLine( + "/usr/bin/cc", + "-shared", + "-o", + "libpkl.dylib", + "$projectDir/src/main/c/libpkl.c", + "-I$projectDir/src/main/c", + "-I$libraryOutputDir", + "-L$libraryOutputDir", + "-lpkl-internal-macos-aarch64", + ) + } + +val macNativeFullLibraryAarch64Copy by + tasks.registering(Exec::class) { + dependsOn(macNativeFullLibraryAarch64) + + val libraryOutputDir = project.layout.buildDirectory.dir("libs/macos-aarch64").get() + val projectDir = project.layout.projectDirectory.asFile.path + + workingDir = libraryOutputDir.asFile + + commandLine("cp", "$projectDir/src/main/c/libpkl.h", libraryOutputDir) + } + +tasks.withType { + dependsOn(macNativeFullLibraryAarch64Copy) + + val nativeLibsDir = project.layout.buildDirectory.dir("libs/macos-aarch64").get().asFile + jvmArgs("-Djna.library.path=${nativeLibsDir.absolutePath}") + + useJUnitPlatform() +} diff --git a/libpkl/src/main/c/libpkl.c b/libpkl/src/main/c/libpkl.c new file mode 100644 index 000000000..4c90ea90e --- /dev/null +++ b/libpkl/src/main/c/libpkl.c @@ -0,0 +1,70 @@ +#include +#include +#include + +#include +#include + +#include + +#ifndef NULL + #define NULL 0 +#endif + +pthread_mutex_t graal_mutex; +graal_isolatethread_t *isolatethread = NULL; + +int pkl_init(PklMessageResponseHandler handler) { + if (isolatethread != NULL) { + perror("pkl_init: isolatethread is already initialised"); + return -1; + } + + if (pthread_mutex_init(&graal_mutex, NULL) != 0) { + perror("pkl_init: couldn't initialise pthread_mutex"); + return -1; + } + + if (pthread_mutex_lock(&graal_mutex) != 0) { + return -1; + } + + isolatethread = pkl_internal_init(); + pkl_internal_register_response_handler(isolatethread, handler); + pkl_internal_server_start(isolatethread); + pthread_mutex_unlock(&graal_mutex); + + return 0; +}; + + +int pkl_send_message(int length, char* message) { + if (pthread_mutex_lock(&graal_mutex) != 0) { + return -1; + } + + pkl_internal_send_message(isolatethread, length, message); + pthread_mutex_unlock(&graal_mutex); + + return 0; +}; + +int pkl_close() { + if (pthread_mutex_lock(&graal_mutex) != 0) { + return -1; + } + + pkl_internal_server_stop(isolatethread); + pkl_internal_close(isolatethread); + isolatethread = NULL; + + if (pthread_mutex_unlock(&graal_mutex) != 0) { + return -1; + } + + if (pthread_mutex_destroy(&graal_mutex) != 0) { + return -1; + } + + return 0; +}; diff --git a/libpkl/src/main/c/libpkl.h b/libpkl/src/main/c/libpkl.h new file mode 100644 index 000000000..f351646d6 --- /dev/null +++ b/libpkl/src/main/c/libpkl.h @@ -0,0 +1,7 @@ +typedef void (*PklMessageResponseHandler)(int length, char* message); + +int pkl_init(PklMessageResponseHandler handler); + +int pkl_send_message(int length, char* message); + +int pkl_close(); diff --git a/libpkl/src/main/java/org/pkl/libpkl/LibPkl.java b/libpkl/src/main/java/org/pkl/libpkl/LibPkl.java new file mode 100644 index 000000000..c2df18852 --- /dev/null +++ b/libpkl/src/main/java/org/pkl/libpkl/LibPkl.java @@ -0,0 +1,89 @@ +/* + * Copyright © 2025 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.libpkl; + +import java.io.IOException; +import org.graalvm.nativeimage.IsolateThread; +import org.graalvm.nativeimage.PinnedObject; +import org.graalvm.nativeimage.c.function.CEntryPoint; +import org.graalvm.nativeimage.c.function.CFunction.Transition; +import org.graalvm.nativeimage.c.function.CFunctionPointer; +import org.graalvm.nativeimage.c.function.InvokeCFunctionPointer; +import org.graalvm.nativeimage.c.type.CCharPointer; +import org.pkl.core.messaging.MessageTransports.Logger; +import org.pkl.core.messaging.ProtocolException; +import org.pkl.server.Server; + +public class LibPkl { + public interface MessageCallbackFunctionPointer extends CFunctionPointer { + @InvokeCFunctionPointer(transition = Transition.TO_NATIVE) + void invoke(int length, CCharPointer msg); + } + + private static final Logger logger = new LibPklLogger(); + private static final NativeTransport transport = + new NativeTransport(logger, LibPkl::handleSendMessageToNative); + private static Server server; + private static MessageCallbackFunctionPointer cb; + + @CEntryPoint(name = "pkl_internal_init", builtin = CEntryPoint.Builtin.CREATE_ISOLATE) + static native IsolateThread pklInternalInit(); + + @CEntryPoint(name = "pkl_internal_send_message") + public static void pklInternalSendMessage(IsolateThread thread, int length, CCharPointer ptr) + throws ProtocolException, IOException { + logger.log("Got message from native"); + transport.sendMessage(length, ptr); + } + + @CEntryPoint(name = "pkl_internal_register_response_handler") + public static void pklInternalRegisterResponseHandler( + IsolateThread thread, LibPkl.MessageCallbackFunctionPointer cb) { + logger.log("Got handler to call from Pkl"); + LibPkl.cb = cb; + } + + @CEntryPoint(name = "pkl_internal_close", builtin = CEntryPoint.Builtin.TEAR_DOWN_ISOLATE) + public static native void pklInternalClose(IsolateThread thread); + + @CEntryPoint(name = "pkl_internal_server_start") + public static void pklInternalServerStart(IsolateThread thread) { + server = new Server(transport); + server.start(); + } + + @CEntryPoint(name = "pkl_internal_server_stop") + public static void pklInternalServerStop(IsolateThread thread) { + server.close(); + } + + public static void handleSendMessageToNative(byte[] bytes) { + try (var pin = PinnedObject.create(bytes)) { + cb.invoke(bytes.length, pin.addressOfArrayElement(0)); + } + } + + /** + * Needed otherwise we see the following error: + * + *

Error: Method 'org.pkl.libpkl.LibPkl.main' is declared as the main entry point but it can + * not be found. Make sure that class 'org.pkl.libpkl.LibPkl' is on the classpath and that method + * 'main(String[])' exists in that class. + */ + public static void main(String[] argv) {} + + private LibPkl() {} +} diff --git a/libpkl/src/main/java/org/pkl/libpkl/LibPklLogger.java b/libpkl/src/main/java/org/pkl/libpkl/LibPklLogger.java new file mode 100644 index 000000000..f6e887d80 --- /dev/null +++ b/libpkl/src/main/java/org/pkl/libpkl/LibPklLogger.java @@ -0,0 +1,29 @@ +/* + * Copyright © 2025 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.libpkl; + +import java.util.Objects; +import org.pkl.core.messaging.MessageTransports.Logger; + +public class LibPklLogger implements Logger { + + @Override + public void log(String msg) { + if (Objects.equals(System.getenv("PKL_DEBUG"), "1")) { + System.err.println("[libpkl] " + msg); + } + } +} diff --git a/libpkl/src/main/java/org/pkl/libpkl/NativeInputStream.java b/libpkl/src/main/java/org/pkl/libpkl/NativeInputStream.java new file mode 100644 index 000000000..ca343575b --- /dev/null +++ b/libpkl/src/main/java/org/pkl/libpkl/NativeInputStream.java @@ -0,0 +1,43 @@ +/* + * Copyright © 2025 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.libpkl; + +import java.io.InputStream; +import org.graalvm.nativeimage.c.type.CCharPointer; + +public class NativeInputStream extends InputStream { + private int offset; + private final int length; + private final CCharPointer ptr; + + @Override + public int read() { + var result = ptr.read(offset); + offset++; + return result; + } + + @Override + public int available() { + return length - offset; + } + + public NativeInputStream(int length, CCharPointer ptr) { + super(); + this.length = length; + this.ptr = ptr; + } +} diff --git a/libpkl/src/main/java/org/pkl/libpkl/NativeTransport.java b/libpkl/src/main/java/org/pkl/libpkl/NativeTransport.java new file mode 100644 index 000000000..400cc0117 --- /dev/null +++ b/libpkl/src/main/java/org/pkl/libpkl/NativeTransport.java @@ -0,0 +1,62 @@ +/* + * Copyright © 2025 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.libpkl; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.function.Consumer; +import org.graalvm.nativeimage.c.type.CCharPointer; +import org.msgpack.core.MessagePack; +import org.pkl.core.messaging.Message; +import org.pkl.core.messaging.MessageTransports.AbstractMessageTransport; +import org.pkl.core.messaging.MessageTransports.Logger; +import org.pkl.core.messaging.ProtocolException; +import org.pkl.server.ServerMessagePackDecoder; +import org.pkl.server.ServerMessagePackEncoder; + +public class NativeTransport extends AbstractMessageTransport { + private final Consumer sendMessageToNative; + + protected NativeTransport(Logger logger, Consumer sendMessageToNative) { + super(logger); + this.sendMessageToNative = sendMessageToNative; + } + + @Override + protected void doStart() throws ProtocolException, IOException {} + + @Override + protected void doClose() {} + + @Override + protected void doSend(Message message) throws IOException, ProtocolException { + try (var os = new ByteArrayOutputStream(); + var packer = MessagePack.newDefaultPacker(os)) { + var encoder = new ServerMessagePackEncoder(packer); + encoder.encode(message); + sendMessageToNative.accept(os.toByteArray()); + } + } + + public void sendMessage(int length, CCharPointer ptr) throws IOException, ProtocolException { + try (var is = new NativeInputStream(length, ptr); + var unpacker = MessagePack.newDefaultUnpacker(is)) { + var message = new ServerMessagePackDecoder(unpacker).decode(); + assert message != null; + accept(message); + } + } +} diff --git a/libpkl/src/test/kotlin/org/pkl/libpkl/ILibPklLibrary.kt b/libpkl/src/test/kotlin/org/pkl/libpkl/ILibPklLibrary.kt new file mode 100644 index 000000000..2271bc4a6 --- /dev/null +++ b/libpkl/src/test/kotlin/org/pkl/libpkl/ILibPklLibrary.kt @@ -0,0 +1,37 @@ +/* + * Copyright © 2025 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.libpkl + +import com.sun.jna.Callback +import com.sun.jna.Library +import com.sun.jna.Native +import com.sun.jna.Pointer + +interface ILibPklLibrary : Library { + companion object { + val INSTANCE: ILibPklLibrary = Native.load("pkl", ILibPklLibrary::class.java) + } + + interface PklMessageResponseHandler : Callback { + fun invoke(length: Int, message: Pointer?) + } + + fun pkl_init(handler: PklMessageResponseHandler?): Int + + fun pkl_send_message(length: Int, message: ByteArray?): Int + + fun pkl_close(): Int +} diff --git a/libpkl/src/test/kotlin/org/pkl/libpkl/JNATest.kt b/libpkl/src/test/kotlin/org/pkl/libpkl/JNATest.kt new file mode 100644 index 000000000..de6678f10 --- /dev/null +++ b/libpkl/src/test/kotlin/org/pkl/libpkl/JNATest.kt @@ -0,0 +1,866 @@ +/* + * Copyright © 2025 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.libpkl + +import java.net.URI +import java.nio.file.Path +import kotlin.io.path.createDirectories +import kotlin.io.path.writeText +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import org.msgpack.core.MessagePack +import org.pkl.commons.test.PackageServer +import org.pkl.core.messaging.Messages.ListModulesRequest +import org.pkl.core.messaging.Messages.ListModulesResponse +import org.pkl.core.messaging.Messages.ListResourcesRequest +import org.pkl.core.messaging.Messages.ListResourcesResponse +import org.pkl.core.messaging.Messages.ModuleReaderSpec +import org.pkl.core.messaging.Messages.ReadModuleRequest +import org.pkl.core.messaging.Messages.ReadModuleResponse +import org.pkl.core.messaging.Messages.ReadResourceRequest +import org.pkl.core.messaging.Messages.ReadResourceResponse +import org.pkl.core.messaging.Messages.ResourceReaderSpec +import org.pkl.core.module.PathElement +import org.pkl.server.* + +/** + * This test cannot be run inside of IntelliJ. + * + * It needs the `jna.library.path` property to be set, which is handled by `libpkl:test`. + */ +class JNATest { + companion object { + lateinit var client: JNATestClient + } + + @BeforeEach + fun beforeEach() { + client = JNATestClient() + assertThat(ILibPklLibrary.INSTANCE.pkl_init(client)).isEqualTo(0) + } + + @AfterEach + fun afterEach() { + client.close() + assertThat(ILibPklLibrary.INSTANCE.pkl_close()).isEqualTo(0) + } + + @Test + fun `create evaluator, receive message, and close evaluator`() { + val evaluatorId = client.sendCreateEvaluatorRequest() + + client.send(EvaluateRequest(1, evaluatorId, URI("repl:text"), """foo = 1""", null)) + assertThat(client).hasSize(1) + + val response = client.receive() + assertThat(response.evaluatorId).isEqualTo(evaluatorId) + + assertThat(client.send(CloseEvaluator(evaluatorId))).isEqualTo(0) + assertThat(client).hasSize(0) + } + + @Test + fun `evaluate module`() { + val evaluatorId = client.sendCreateEvaluatorRequest() + val requestId = 234L + + client.send( + EvaluateRequest( + requestId, + evaluatorId, + URI("repl:text"), + """ + foo { + bar = "bar" + } + """ + .trimIndent(), + null, + ) + ) + + val response = client.receive() + assertThat(response.error).isNull() + assertThat(response.result).isNotNull + assertThat(response.requestId()).isEqualTo(requestId) + + val unpacker = MessagePack.newDefaultUnpacker(response.result) + val value = unpacker.unpackValue() + assertThat(value.isArrayValue) + } + + @Test + fun `trace logs`() { + val evaluatorId = client.sendCreateEvaluatorRequest() + + client.send( + EvaluateRequest( + 1, + evaluatorId, + URI("repl:text"), + """ + foo = trace(1 + 2 + 3) + """ + .trimIndent(), + null, + ) + ) + + val response = client.receive() + assertThat(response.level).isEqualTo(0) + assertThat(response.message).isEqualTo("1 + 2 + 3 = 6") + + // client.receive() + } + + @Test + fun `warn logs`() { + val evaluatorId = client.sendCreateEvaluatorRequest() + + client.send( + EvaluateRequest( + 1, + evaluatorId, + URI("repl:text"), + """ + @Deprecated { message = "use bar instead" } + function foo() = 5 + + result = foo() + """ + .trimIndent(), + null, + ) + ) + + val response = client.receive() + assertThat(response.level).isEqualTo(1) + assertThat(response.message).contains("use bar instead") + + client.receive() + } + + @Test + fun `read resource`() { + val reader = ResourceReaderSpec("bahumbug", true, false) + val evaluatorId = client.sendCreateEvaluatorRequest(resourceReaders = listOf(reader)) + + client.send( + EvaluateRequest( + 1, + evaluatorId, + URI("repl:text"), + """res = read("bahumbug:/foo.pkl").text""", + "res", + ) + ) + + val readResourceMsg = client.receive() + assertThat(readResourceMsg.uri.toString()).isEqualTo("bahumbug:/foo.pkl") + assertThat(readResourceMsg.evaluatorId).isEqualTo(evaluatorId) + + client.send( + ReadResourceResponse( + readResourceMsg.requestId, + evaluatorId, + "my bahumbug".toByteArray(), + null, + ) + ) + + val evaluateResponse = client.receive() + assertThat(evaluateResponse.error).isNull() + + val unpacker = MessagePack.newDefaultUnpacker(evaluateResponse.result) + val value = unpacker.unpackValue() + assertThat(value.asStringValue().asString()).isEqualTo("my bahumbug") + } + + @Test + fun `read resource error`() { + val reader = ResourceReaderSpec("bahumbug", true, false) + val evaluatorId = client.sendCreateEvaluatorRequest(resourceReaders = listOf(reader)) + + client.send( + EvaluateRequest( + 1, + evaluatorId, + URI("repl:text"), + """res = read("bahumbug:/foo.txt").text""", + "res", + ) + ) + + val readResourceMsg = client.receive() + + client.send( + ReadResourceResponse( + readResourceMsg.requestId, + evaluatorId, + byteArrayOf(), + "cannot read my bahumbug", + ) + ) + + val evaluateResponse = client.receive() + assertThat(evaluateResponse.error).contains("bahumbug:/foo.txt") + assertThat(evaluateResponse.error).doesNotContain("org.pkl.core.PklBugException") + } + + @Test + fun `glob resource`() { + val reader = ResourceReaderSpec("bird", true, true) + val evaluatorId = client.sendCreateEvaluatorRequest(resourceReaders = listOf(reader)) + client.send( + EvaluateRequest( + 1, + evaluatorId, + URI("repl:text"), + """ + res = read*("bird:/**.txt").keys + """ + .trimIndent(), + "res", + ) + ) + val listResourcesRequest = client.receive() + assertThat(listResourcesRequest.uri.toString()).isEqualTo("bird:/") + client.send( + ListResourcesResponse( + listResourcesRequest.requestId, + listResourcesRequest.evaluatorId, + listOf(PathElement("foo.txt", false), PathElement("subdir", true)), + null, + ) + ) + val listResourcesRequest2 = client.receive() + assertThat(listResourcesRequest2.uri.toString()).isEqualTo("bird:/subdir/") + client.send( + ListResourcesResponse( + listResourcesRequest2.requestId, + listResourcesRequest2.evaluatorId, + listOf(PathElement("bar.txt", false)), + null, + ) + ) + val evaluateResponse = client.receive() + assertThat(evaluateResponse.result?.debugYaml) + .isEqualTo( + """ + - 6 + - + - bird:/foo.txt + - bird:/subdir/bar.txt + """ + .trimIndent() + ) + } + + @Test + fun `glob resources -- null pathElements and null error`() { + val reader = ResourceReaderSpec("bird", true, true) + val evaluatorId = client.sendCreateEvaluatorRequest(resourceReaders = listOf(reader)) + client.send( + EvaluateRequest( + 1, + evaluatorId, + URI("repl:text"), + """ + res = read*("bird:/**.txt").keys + """ + .trimIndent(), + "res", + ) + ) + val listResourcesRequest = client.receive() + client.send( + ListResourcesResponse( + listResourcesRequest.requestId, + listResourcesRequest.evaluatorId, + null, + null, + ) + ) + val evaluateResponse = client.receive() + assertThat(evaluateResponse.result?.debugYaml) + .isEqualTo( + """ + - 6 + - [] + """ + .trimIndent() + ) + } + + @Test + fun `glob resource error`() { + val reader = ResourceReaderSpec("bird", true, true) + val evaluatorId = client.sendCreateEvaluatorRequest(resourceReaders = listOf(reader)) + client.send( + EvaluateRequest( + 1, + evaluatorId, + URI("repl:text"), + """ + res = read*("bird:/**.txt").keys + """ + .trimIndent(), + "res", + ) + ) + val listResourcesRequest = client.receive() + assertThat(listResourcesRequest.uri.toString()).isEqualTo("bird:/") + client.send( + ListResourcesResponse( + listResourcesRequest.requestId, + listResourcesRequest.evaluatorId, + null, + "didnt work", + ) + ) + val evaluateResponse = client.receive() + assertThat(evaluateResponse.error) + .isEqualTo( + """ + –– Pkl Error –– + I/O error resolving glob pattern `bird:/**.txt`. + IOException: didnt work + + 1 | res = read*("bird:/**.txt").keys + ^^^^^^^^^^^^^^^^^^^^^ + at text#res (repl:text) + + 1 | res + ^^^ + at (repl:text) + + """ + .trimIndent() + ) + } + + @Test + fun `read module`() { + val reader = ModuleReaderSpec("bird", true, true, false) + val evaluatorId = client.sendCreateEvaluatorRequest(moduleReaders = listOf(reader)) + + client.send( + EvaluateRequest( + 1, + evaluatorId, + URI("repl:text"), + """res = import("bird:/pigeon.pkl").value""", + "res", + ) + ) + + val readModuleMsg = client.receive() + assertThat(readModuleMsg.uri.toString()).isEqualTo("bird:/pigeon.pkl") + assertThat(readModuleMsg.evaluatorId).isEqualTo(evaluatorId) + + client.send(ReadModuleResponse(readModuleMsg.requestId, evaluatorId, "value = 5", null)) + + val evaluateResponse = client.receive() + assertThat(evaluateResponse.error).isNull() + val unpacker = MessagePack.newDefaultUnpacker(evaluateResponse.result) + val value = unpacker.unpackValue() + assertThat(value.asIntegerValue().asInt()).isEqualTo(5) + } + + @Test + fun `read module -- null contents and null error`() { + val reader = ModuleReaderSpec("bird", true, true, false) + val evaluatorId = client.sendCreateEvaluatorRequest(moduleReaders = listOf(reader)) + + client.send( + EvaluateRequest( + requestId = 1, + evaluatorId = evaluatorId, + moduleUri = URI("repl:text"), + moduleText = """res = import("bird:/pigeon.pkl")""", + expr = "res", + ) + ) + + val readModuleMsg = client.receive() + assertThat(readModuleMsg.uri.toString()).isEqualTo("bird:/pigeon.pkl") + assertThat(readModuleMsg.evaluatorId).isEqualTo(evaluatorId) + + client.send(ReadModuleResponse(readModuleMsg.requestId, evaluatorId, null, null)) + + val evaluateResponse = client.receive() + assertThat(evaluateResponse.error).isNull() + val unpacker = MessagePack.newDefaultUnpacker(evaluateResponse.result) + val value = unpacker.unpackValue().asArrayValue().list() + assertThat(value[0].asIntegerValue().asLong()).isEqualTo(0x1) + assertThat(value[1].asStringValue().asString()).isEqualTo("pigeon") + assertThat(value[3].asArrayValue().list()).isEmpty() + } + + @Test + fun `read module error`() { + val reader = ModuleReaderSpec("bird", true, true, false) + val evaluatorId = client.sendCreateEvaluatorRequest(moduleReaders = listOf(reader)) + + client.send( + EvaluateRequest( + 1, + evaluatorId, + URI("repl:text"), + """res = import("bird:/pigeon.pkl").value""", + "res", + ) + ) + + val readModuleMsg = client.receive() + assertThat(readModuleMsg.uri.toString()).isEqualTo("bird:/pigeon.pkl") + assertThat(readModuleMsg.evaluatorId).isEqualTo(evaluatorId) + + client.send( + ReadModuleResponse(readModuleMsg.requestId, evaluatorId, null, "Don't know where Pigeon is") + ) + + val evaluateResponse = client.receive() + assertThat(evaluateResponse.error).contains("Don't know where Pigeon is") + } + + @Test + fun `glob module`() { + val reader = ModuleReaderSpec("bird", true, true, true) + val evaluatorId = client.sendCreateEvaluatorRequest(moduleReaders = listOf(reader)) + + client.send( + EvaluateRequest( + 1, + evaluatorId, + URI("repl:text"), + """res = import*("bird:/**.pkl").keys""", + "res", + ) + ) + + val listModulesMsg = client.receive() + assertThat(listModulesMsg.uri.scheme).isEqualTo("bird") + assertThat(listModulesMsg.uri.path).isEqualTo("/") + client.send( + ListModulesResponse( + listModulesMsg.requestId, + evaluatorId, + listOf( + PathElement("birds", true), + PathElement("majesticBirds", true), + PathElement("Person.pkl", false), + ), + null, + ) + ) + val listModulesMsg2 = client.receive() + assertThat(listModulesMsg2.uri.scheme).isEqualTo("bird") + assertThat(listModulesMsg2.uri.path).isEqualTo("/birds/") + client.send( + ListModulesResponse( + listModulesMsg2.requestId, + listModulesMsg2.evaluatorId, + listOf(PathElement("pigeon.pkl", false), PathElement("parrot.pkl", false)), + null, + ) + ) + val listModulesMsg3 = client.receive() + assertThat(listModulesMsg3.uri.scheme).isEqualTo("bird") + assertThat(listModulesMsg3.uri.path).isEqualTo("/majesticBirds/") + client.send( + ListModulesResponse( + listModulesMsg3.requestId, + listModulesMsg3.evaluatorId, + listOf(PathElement("barnOwl.pkl", false), PathElement("elfOwl.pkl", false)), + null, + ) + ) + + val evaluateResponse = client.receive() + assertThat(evaluateResponse.result?.debugRendering) + .isEqualTo( + """ + - 6 + - + - bird:/Person.pkl + - bird:/birds/parrot.pkl + - bird:/birds/pigeon.pkl + - bird:/majesticBirds/barnOwl.pkl + - bird:/majesticBirds/elfOwl.pkl + """ + .trimIndent() + ) + } + + @Test + fun `glob module -- null pathElements and null error`() { + val reader = ModuleReaderSpec("bird", true, true, true) + val evaluatorId = client.sendCreateEvaluatorRequest(moduleReaders = listOf(reader)) + + client.send( + EvaluateRequest( + 1, + evaluatorId, + URI("repl:text"), + """res = import*("bird:/**.pkl").keys""", + "res", + ) + ) + val listModulesMsg = client.receive() + client.send(ListModulesResponse(listModulesMsg.requestId, evaluatorId, null, null)) + + val evaluateResponse = client.receive() + assertThat(evaluateResponse.result?.debugRendering) + .isEqualTo( + """ + - 6 + - [] + """ + .trimIndent() + ) + } + + @Test + fun `glob module error`() { + val reader = ModuleReaderSpec("bird", true, true, true) + val evaluatorId = client.sendCreateEvaluatorRequest(moduleReaders = listOf(reader)) + + client.send( + EvaluateRequest( + 1, + evaluatorId, + URI("repl:text"), + """res = import*("bird:/**.pkl").keys""", + "res", + ) + ) + + val listModulesMsg = client.receive() + assertThat(listModulesMsg.uri.scheme).isEqualTo("bird") + assertThat(listModulesMsg.uri.path).isEqualTo("/") + client.send(ListModulesResponse(listModulesMsg.requestId, evaluatorId, null, "nope")) + val evaluateResponse = client.receive() + assertThat(evaluateResponse.error) + .isEqualTo( + """ + –– Pkl Error –– + I/O error resolving glob pattern `bird:/**.pkl`. + IOException: nope + + 1 | res = import*("bird:/**.pkl").keys + ^^^^^^^^^^^^^^^^^^^^^^^ + at text#res (repl:text) + + 1 | res + ^^^ + at (repl:text) + + """ + .trimIndent() + ) + } + + @Test + fun `import triple-dot path`() { + val reader = ModuleReaderSpec("bird", true, true, true) + val evaluatorId = client.sendCreateEvaluatorRequest(moduleReaders = listOf(reader)) + + client.send( + EvaluateRequest( + 1, + evaluatorId, + URI("bird:/foo/bar/baz.pkl"), + """ + import ".../buz.pkl" + + res = buz.res + """ + .trimIndent(), + "res", + ) + ) + val readModuleRequest = client.receive() + assertThat(readModuleRequest.uri).isEqualTo(URI("bird:/foo/buz.pkl")) + client.send( + ReadModuleResponse( + readModuleRequest.requestId, + readModuleRequest.evaluatorId, + null, + "not here", + ) + ) + + val readModuleRequest2 = client.receive() + assertThat(readModuleRequest2.uri).isEqualTo(URI("bird:/buz.pkl")) + client.send( + ReadModuleResponse( + readModuleRequest2.requestId, + readModuleRequest2.evaluatorId, + "res = 1", + null, + ) + ) + + val evaluatorResponse = client.receive() + assertThat(evaluatorResponse.result?.debugYaml).isEqualTo("1") + } + + @Test + fun `evaluate error`() { + val evaluatorId = client.sendCreateEvaluatorRequest() + + client.send(EvaluateRequest(1, evaluatorId, URI("repl:text"), """foo = 1""", "foo as String")) + + val evaluateResponse = client.receive() + assertThat(evaluateResponse.requestId()).isEqualTo(1) + assertThat(evaluateResponse.error).contains("Expected value of type") + } + + @Test + fun `evaluate client-provided module reader`() { + val reader = ModuleReaderSpec("bird", true, false, false) + val evaluatorId = client.sendCreateEvaluatorRequest(moduleReaders = listOf(reader)) + + client.send(EvaluateRequest(1, evaluatorId, URI("bird:/pigeon.pkl"), null, "output.text")) + + val readModuleRequest = client.receive() + assertThat(readModuleRequest.uri.toString()).isEqualTo("bird:/pigeon.pkl") + + client.send( + ReadModuleResponse( + readModuleRequest.requestId, + evaluatorId, + """ + firstName = "Pigeon" + lastName = "Bird" + fullName = firstName + " " + lastName + """ + .trimIndent(), + null, + ) + ) + + val evaluateResponse = client.receive() + assertThat(evaluateResponse.result).isNotNull + assertThat(evaluateResponse.result?.debugYaml) + .isEqualTo( + """ + | + firstName = "Pigeon" + lastName = "Bird" + fullName = "Pigeon Bird" + """ + .trimIndent() + ) + } + + @Test + fun `concurrent evaluations`() { + val reader = ModuleReaderSpec("bird", true, false, false) + val evaluatorId = client.sendCreateEvaluatorRequest(moduleReaders = listOf(reader)) + client.send(EvaluateRequest(1, evaluatorId, URI("bird:/pigeon.pkl"), null, "output.text")) + + client.send(EvaluateRequest(2, evaluatorId, URI("bird:/parrot.pkl"), null, "output.text")) + + // evaluation is single-threaded; `parrot.pkl` gets evaluated after `pigeon.pkl` completes. + val response11 = client.receive() + assertThat(response11.uri.toString()).isEqualTo("bird:/pigeon.pkl") + + client.send( + ReadModuleResponse( + response11.requestId, + evaluatorId, + """ + firstName = "Pigeon" + lastName = "Bird" + fullName = firstName + " " + lastName + """ + .trimIndent(), + null, + ) + ) + + val response12 = client.receive() + assertThat(response12.result).isNotNull + assertThat(response12.result?.debugYaml) + .isEqualTo( + """ + | + firstName = "Pigeon" + lastName = "Bird" + fullName = "Pigeon Bird" + """ + .trimIndent() + ) + + val response21 = client.receive() + assertThat(response21.uri.toString()).isEqualTo("bird:/parrot.pkl") + + client.send( + ReadModuleResponse( + response21.requestId, + evaluatorId, + """ + firstName = "Parrot" + lastName = "Bird" + fullName = firstName + " " + lastName + """ + .trimIndent(), + null, + ) + ) + + val response22 = client.receive() + assertThat(response22.result).isNotNull + assertThat(response22.result?.debugYaml) + .isEqualTo( + """ + | + firstName = "Parrot" + lastName = "Bird" + fullName = "Parrot Bird" + """ + .trimIndent() + ) + } + + @Test + fun `evaluate with project dependencies`(@TempDir tempDir: Path) { + val cacheDir = tempDir.resolve("cache").createDirectories() + PackageServer.populateCacheDir(cacheDir) + val libDir = tempDir.resolve("lib/").createDirectories() + libDir + .resolve("lib.pkl") + .writeText( + """ + text = "This is from lib" + """ + .trimIndent() + ) + libDir + .resolve("PklProject") + .writeText( + """ + amends "pkl:Project" + + package { + name = "lib" + baseUri = "package://localhost:0/lib" + version = "5.0.0" + packageZipUrl = "https://localhost:0/lib.zip" + } + """ + .trimIndent() + ) + val projectDir = tempDir.resolve("proj/").createDirectories() + val module = projectDir.resolve("mod.pkl") + module.writeText( + """ + import "@birds/Bird.pkl" + import "@lib/lib.pkl" + + res: Bird = new { + name = "Birdie" + favoriteFruit { name = "dragonfruit" } + } + + libContents = lib + """ + .trimIndent() + ) + val dollar = '$' + projectDir + .resolve("PklProject.deps.json") + .writeText( + """ + { + "schemaVersion": 1, + "resolvedDependencies": { + "package://localhost:0/birds@0": { + "type": "remote", + "uri": "projectpackage://localhost:0/birds@0.5.0", + "checksums": { + "sha256": "${dollar}skipChecksumVerification" + } + }, + "package://localhost:0/fruit@1": { + "type": "remote", + "uri": "projectpackage://localhost:0/fruit@1.0.5", + "checksums": { + "sha256": "${dollar}skipChecksumVerification" + } + }, + "package://localhost:0/lib@5": { + "type": "local", + "uri": "projectpackage://localhost:0/lib@5.0.0", + "path": "../lib" + } + } + } + + """ + .trimIndent() + ) + val evaluatorId = + client.sendCreateEvaluatorRequest( + cacheDir = cacheDir, + project = + Project( + projectDir.resolve("PklProject").toUri(), + null, + mapOf( + "birds" to RemoteDependency(URI("package://localhost:0/birds@0.5.0"), null), + "lib" to + Project( + libDir.toUri().resolve("PklProject"), + URI("package://localhost:0/lib@5.0.0"), + emptyMap(), + ), + ), + ), + ) + client.send(EvaluateRequest(1, evaluatorId, module.toUri(), null, "output.text")) + val resp2 = client.receive() + assertThat(resp2.error).isNull() + assertThat(resp2.result).isNotNull() + assertThat(resp2.result?.debugRendering?.trim()) + .isEqualTo( + """ + | + res { + name = "Birdie" + favoriteFruit { + name = "dragonfruit" + } + } + libContents { + text = "This is from lib" + } + """ + .trimIndent() + ) + } + + private val ByteArray.debugYaml + get() = MessagePackDebugRenderer(this).output.trimIndent() + + private val ByteArray.debugRendering: String + get() = MessagePackDebugRenderer(this).output +} diff --git a/libpkl/src/test/kotlin/org/pkl/libpkl/JNATestClient.kt b/libpkl/src/test/kotlin/org/pkl/libpkl/JNATestClient.kt new file mode 100644 index 000000000..31a4cc3fc --- /dev/null +++ b/libpkl/src/test/kotlin/org/pkl/libpkl/JNATestClient.kt @@ -0,0 +1,116 @@ +/* + * Copyright © 2025 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.libpkl + +import com.sun.jna.Pointer +import java.io.ByteArrayOutputStream +import java.lang.AutoCloseable +import java.nio.file.Path +import java.util.concurrent.ArrayBlockingQueue +import java.util.concurrent.BlockingQueue +import org.assertj.core.api.Assertions.assertThat +import org.msgpack.core.MessagePack +import org.pkl.core.messaging.Message +import org.pkl.core.messaging.Messages.ModuleReaderSpec +import org.pkl.core.messaging.Messages.ResourceReaderSpec +import org.pkl.server.CreateEvaluatorRequest +import org.pkl.server.CreateEvaluatorResponse +import org.pkl.server.Http +import org.pkl.server.Project +import org.pkl.server.ServerMessagePackDecoder +import org.pkl.server.ServerMessagePackEncoder + +class JNATestClient : ILibPklLibrary.PklMessageResponseHandler, Iterable, AutoCloseable { + val incoming: BlockingQueue = ArrayBlockingQueue(10) + + override fun invoke(length: Int, message: Pointer?) { + when { + message != null && length > 0 -> { + val receivedBytes: ByteArray = message.getByteArray(0, length) + val message = decode(receivedBytes) + assertThat(message).isInstanceOf(Message::class.java) + incoming.add(message!!) + } + else -> incoming.add(null) + } + } + + override fun close() = incoming.clear() + + override fun iterator(): Iterator = incoming.iterator() + + fun send(message: Message): Int = + encode(message).let { ILibPklLibrary.INSTANCE.pkl_send_message(it.size, it) } + + inline fun receive(): T { + val message = incoming.take() + assertThat(message).isInstanceOf(T::class.java) + return message as T + } + + fun sendCreateEvaluatorRequest( + requestId: Long = 123, + resourceReaders: List = listOf(), + moduleReaders: List = listOf(), + modulePaths: List = listOf(), + project: Project? = null, + cacheDir: Path? = null, + http: Http? = null, + ): Long { + val message = + CreateEvaluatorRequest( + 123, + listOf(".*"), + listOf(".*"), + moduleReaders, + resourceReaders, + modulePaths, + mapOf(), + mapOf(), + null, + null, + cacheDir, + null, + project, + http, + null, + null, + ) + + send(message) + + val response = receive() + assertThat(response.requestId()).isEqualTo(requestId) + assertThat(response.evaluatorId).isNotNull + assertThat(response.error).isNull() + + return response.evaluatorId!! + } + + private fun encode(message: Message): ByteArray { + ByteArrayOutputStream().use { os -> + val packer = MessagePack.newDefaultPacker(os) + val encoder = ServerMessagePackEncoder(packer) + encoder.encode(message) + return os.toByteArray() + } + } + + private fun decode(receivedBytes: ByteArray): Message? { + val unpacker = MessagePack.newDefaultUnpacker(receivedBytes) + return ServerMessagePackDecoder(unpacker).decode() + } +} diff --git a/libpkl/src/test/kotlin/org/pkl/libpkl/MessagePackDebugRenderer.kt b/libpkl/src/test/kotlin/org/pkl/libpkl/MessagePackDebugRenderer.kt new file mode 100644 index 000000000..8d4d01d4e --- /dev/null +++ b/libpkl/src/test/kotlin/org/pkl/libpkl/MessagePackDebugRenderer.kt @@ -0,0 +1,107 @@ +/* + * Copyright © 2025 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.libpkl + +import java.lang.IllegalStateException +import org.msgpack.core.MessagePack +import org.msgpack.core.MessageUnpacker +import org.msgpack.value.ValueType +import org.pkl.core.util.yaml.YamlEmitter + +/** Renders MessagePack structures in YAML. */ +class MessagePackDebugRenderer(bytes: ByteArray) { + private val unpacker: MessageUnpacker = MessagePack.newDefaultUnpacker(bytes) + private val currIndent = StringBuilder("") + private val sb = StringBuilder() + private val indent = " " + private val yamlEmitter = YamlEmitter.create(sb, "1.2", indent) + + private fun incIndent() { + currIndent.append(indent) + } + + private fun decIndent() { + currIndent.setLength(currIndent.length - indent.length) + } + + private fun newline() { + sb.append("\n") + sb.append(currIndent) + } + + private fun renderKey() { + val mf = unpacker.nextFormat + when (mf.valueType!!) { + ValueType.STRING -> yamlEmitter.emit(unpacker.unpackString(), currIndent, true) + ValueType.MAP, + ValueType.ARRAY -> { + sb.append("? ") + incIndent() + renderValue() + decIndent() + newline() + } + else -> renderValue() + } + sb.append(": ") + } + + private fun renderValue() { + val mf = unpacker.nextFormat + when (mf.valueType!!) { + ValueType.INTEGER, + ValueType.FLOAT, + ValueType.BOOLEAN, + ValueType.NIL -> sb.append(unpacker.unpackValue().toJson()) + ValueType.STRING -> yamlEmitter.emit(unpacker.unpackString(), currIndent, false) + ValueType.ARRAY -> { + val size = unpacker.unpackArrayHeader() + if (size == 0) { + sb.append("[]") + return + } + for (i in 0 until size) { + newline() + sb.append("- ") + incIndent() + renderValue() + decIndent() + } + } + ValueType.MAP -> { + val size = unpacker.unpackMapHeader() + if (size == 0) { + sb.append("{}") + return + } + for (i in 0 until size) { + newline() + renderKey() + incIndent() + renderValue() + decIndent() + } + } + ValueType.BINARY, + ValueType.EXTENSION -> throw IllegalStateException("Unexpected value type ${mf.valueType}") + } + } + + val output: String by lazy { + renderValue() + sb.toString().removePrefix("\n") + } +} diff --git a/pkl-core/src/main/java/org/pkl/core/messaging/MessageTransports.java b/pkl-core/src/main/java/org/pkl/core/messaging/MessageTransports.java index 7062638a8..9a8e65b35 100644 --- a/pkl-core/src/main/java/org/pkl/core/messaging/MessageTransports.java +++ b/pkl-core/src/main/java/org/pkl/core/messaging/MessageTransports.java @@ -1,5 +1,5 @@ /* - * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. + * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -120,7 +120,7 @@ public void setOther(DirectMessageTransport other) { } } - protected abstract static class AbstractMessageTransport implements MessageTransport { + public abstract static class AbstractMessageTransport implements MessageTransport { private final Logger logger; private MessageTransport.OneWayHandler oneWayHandler; diff --git a/settings.gradle.kts b/settings.gradle.kts index ccac20b85..717957855 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -19,6 +19,8 @@ include("bench") include("docs") +include("libpkl") + include("pkl-cli") include("pkl-codegen-java") From dc85b5200c3fdda1151f4d42f8bcf58b5b90917d Mon Sep 17 00:00:00 2001 From: Kushal Pisavadia Date: Wed, 7 May 2025 11:31:17 +0100 Subject: [PATCH 02/14] Rename to `LibPklLibrary` --- libpkl/src/test/kotlin/org/pkl/libpkl/JNATest.kt | 4 ++-- libpkl/src/test/kotlin/org/pkl/libpkl/JNATestClient.kt | 4 ++-- .../org/pkl/libpkl/{ILibPklLibrary.kt => LibPklLibrary.kt} | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) rename libpkl/src/test/kotlin/org/pkl/libpkl/{ILibPklLibrary.kt => LibPklLibrary.kt} (89%) diff --git a/libpkl/src/test/kotlin/org/pkl/libpkl/JNATest.kt b/libpkl/src/test/kotlin/org/pkl/libpkl/JNATest.kt index de6678f10..707217f6d 100644 --- a/libpkl/src/test/kotlin/org/pkl/libpkl/JNATest.kt +++ b/libpkl/src/test/kotlin/org/pkl/libpkl/JNATest.kt @@ -52,13 +52,13 @@ class JNATest { @BeforeEach fun beforeEach() { client = JNATestClient() - assertThat(ILibPklLibrary.INSTANCE.pkl_init(client)).isEqualTo(0) + assertThat(LibPklLibrary.INSTANCE.pkl_init(client)).isEqualTo(0) } @AfterEach fun afterEach() { client.close() - assertThat(ILibPklLibrary.INSTANCE.pkl_close()).isEqualTo(0) + assertThat(LibPklLibrary.INSTANCE.pkl_close()).isEqualTo(0) } @Test diff --git a/libpkl/src/test/kotlin/org/pkl/libpkl/JNATestClient.kt b/libpkl/src/test/kotlin/org/pkl/libpkl/JNATestClient.kt index 31a4cc3fc..8e7737137 100644 --- a/libpkl/src/test/kotlin/org/pkl/libpkl/JNATestClient.kt +++ b/libpkl/src/test/kotlin/org/pkl/libpkl/JNATestClient.kt @@ -33,7 +33,7 @@ import org.pkl.server.Project import org.pkl.server.ServerMessagePackDecoder import org.pkl.server.ServerMessagePackEncoder -class JNATestClient : ILibPklLibrary.PklMessageResponseHandler, Iterable, AutoCloseable { +class JNATestClient : LibPklLibrary.PklMessageResponseHandler, Iterable, AutoCloseable { val incoming: BlockingQueue = ArrayBlockingQueue(10) override fun invoke(length: Int, message: Pointer?) { @@ -53,7 +53,7 @@ class JNATestClient : ILibPklLibrary.PklMessageResponseHandler, Iterable = incoming.iterator() fun send(message: Message): Int = - encode(message).let { ILibPklLibrary.INSTANCE.pkl_send_message(it.size, it) } + encode(message).let { LibPklLibrary.INSTANCE.pkl_send_message(it.size, it) } inline fun receive(): T { val message = incoming.take() diff --git a/libpkl/src/test/kotlin/org/pkl/libpkl/ILibPklLibrary.kt b/libpkl/src/test/kotlin/org/pkl/libpkl/LibPklLibrary.kt similarity index 89% rename from libpkl/src/test/kotlin/org/pkl/libpkl/ILibPklLibrary.kt rename to libpkl/src/test/kotlin/org/pkl/libpkl/LibPklLibrary.kt index 2271bc4a6..a3e417ad7 100644 --- a/libpkl/src/test/kotlin/org/pkl/libpkl/ILibPklLibrary.kt +++ b/libpkl/src/test/kotlin/org/pkl/libpkl/LibPklLibrary.kt @@ -20,9 +20,9 @@ import com.sun.jna.Library import com.sun.jna.Native import com.sun.jna.Pointer -interface ILibPklLibrary : Library { +interface LibPklLibrary : Library { companion object { - val INSTANCE: ILibPklLibrary = Native.load("pkl", ILibPklLibrary::class.java) + val INSTANCE: LibPklLibrary = Native.load("pkl", LibPklLibrary::class.java) } interface PklMessageResponseHandler : Callback { From 687690797355fbe03aa1f6fe6e2dd25fb23d25db Mon Sep 17 00:00:00 2001 From: Kushal Pisavadia Date: Wed, 7 May 2025 11:33:14 +0100 Subject: [PATCH 03/14] Remove `lib` prefix from `.h` and `.c` files --- libpkl/libpkl.gradle.kts | 4 ++-- libpkl/src/main/c/{libpkl.c => pkl.c} | 2 +- libpkl/src/main/c/{libpkl.h => pkl.h} | 0 3 files changed, 3 insertions(+), 3 deletions(-) rename libpkl/src/main/c/{libpkl.c => pkl.c} (98%) rename libpkl/src/main/c/{libpkl.h => pkl.h} (100%) diff --git a/libpkl/libpkl.gradle.kts b/libpkl/libpkl.gradle.kts index 37fcbaba5..bb2960cd2 100644 --- a/libpkl/libpkl.gradle.kts +++ b/libpkl/libpkl.gradle.kts @@ -289,7 +289,7 @@ val macNativeFullLibraryAarch64 by "-shared", "-o", "libpkl.dylib", - "$projectDir/src/main/c/libpkl.c", + "$projectDir/src/main/c/pkl.c", "-I$projectDir/src/main/c", "-I$libraryOutputDir", "-L$libraryOutputDir", @@ -306,7 +306,7 @@ val macNativeFullLibraryAarch64Copy by workingDir = libraryOutputDir.asFile - commandLine("cp", "$projectDir/src/main/c/libpkl.h", libraryOutputDir) + commandLine("cp", "$projectDir/src/main/c/pkl.h", libraryOutputDir) } tasks.withType { diff --git a/libpkl/src/main/c/libpkl.c b/libpkl/src/main/c/pkl.c similarity index 98% rename from libpkl/src/main/c/libpkl.c rename to libpkl/src/main/c/pkl.c index 4c90ea90e..ed038dcf5 100644 --- a/libpkl/src/main/c/libpkl.c +++ b/libpkl/src/main/c/pkl.c @@ -5,7 +5,7 @@ #include #include -#include +#include #ifndef NULL #define NULL 0 diff --git a/libpkl/src/main/c/libpkl.h b/libpkl/src/main/c/pkl.h similarity index 100% rename from libpkl/src/main/c/libpkl.h rename to libpkl/src/main/c/pkl.h From 5a9e685cb9274640606b804c3cd95440f7c092a1 Mon Sep 17 00:00:00 2001 From: Kushal Pisavadia Date: Wed, 7 May 2025 11:37:20 +0100 Subject: [PATCH 04/14] Add doxygen-style comments to `pkl.h` file --- libpkl/src/main/c/pkl.h | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/libpkl/src/main/c/pkl.h b/libpkl/src/main/c/pkl.h index f351646d6..a41aa36a2 100644 --- a/libpkl/src/main/c/pkl.h +++ b/libpkl/src/main/c/pkl.h @@ -1,7 +1,30 @@ +/** +* @brief The Pkl Message Response Handler that a user should implement. +* +* The resulting messages from `pkl_send` will be sent to this handler using a callback style. +*/ typedef void (*PklMessageResponseHandler)(int length, char* message); +/** +* @brief Initialises and allocates a Pkl executor. +* +* @return -1 on failure. +* @return 0 on success. +*/ int pkl_init(PklMessageResponseHandler handler); +/** +* @brief Send a message to Pkl, providing the length and a pointer to the first byte. +* +* @return -1 on failure. +* @return 0 on success. +*/ int pkl_send_message(int length, char* message); +/** +* @brief Cleans up any resources that were created as part of the `pkl_init` process. +* +* @return -1 on failure. +* @return 0 on success. +*/ int pkl_close(); From e44e441d0e5d70bb1b5a6d7c92d9c4b7b9b67173 Mon Sep 17 00:00:00 2001 From: Kushal Pisavadia Date: Wed, 7 May 2025 11:37:39 +0100 Subject: [PATCH 05/14] Move pointer next to variable name --- libpkl/src/main/c/pkl.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libpkl/src/main/c/pkl.h b/libpkl/src/main/c/pkl.h index a41aa36a2..0e4a9deef 100644 --- a/libpkl/src/main/c/pkl.h +++ b/libpkl/src/main/c/pkl.h @@ -3,7 +3,7 @@ * * The resulting messages from `pkl_send` will be sent to this handler using a callback style. */ -typedef void (*PklMessageResponseHandler)(int length, char* message); +typedef void (*PklMessageResponseHandler)(int length, char *message); /** * @brief Initialises and allocates a Pkl executor. @@ -19,7 +19,7 @@ int pkl_init(PklMessageResponseHandler handler); * @return -1 on failure. * @return 0 on success. */ -int pkl_send_message(int length, char* message); +int pkl_send_message(int length, char *message); /** * @brief Cleans up any resources that were created as part of the `pkl_init` process. From f0c4176de1e941fd67df522399bb721bff5dee36 Mon Sep 17 00:00:00 2001 From: Kushal Pisavadia Date: Wed, 7 May 2025 11:40:50 +0100 Subject: [PATCH 06/14] Add TODO comment to clean this up once on a feature-branch --- libpkl/src/main/java/org/pkl/libpkl/LibPkl.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/libpkl/src/main/java/org/pkl/libpkl/LibPkl.java b/libpkl/src/main/java/org/pkl/libpkl/LibPkl.java index c2df18852..e2c95909b 100644 --- a/libpkl/src/main/java/org/pkl/libpkl/LibPkl.java +++ b/libpkl/src/main/java/org/pkl/libpkl/LibPkl.java @@ -82,6 +82,11 @@ public static void handleSendMessageToNative(byte[] bytes) { *

Error: Method 'org.pkl.libpkl.LibPkl.main' is declared as the main entry point but it can * not be found. Make sure that class 'org.pkl.libpkl.LibPkl' is on the classpath and that method * 'main(String[])' exists in that class. + * + *

TODO: Clean this up once merged onto a feature-branch + * + *

This is because we are passing a main class to native-image using the -H:Class= arg. That + * argument is optional and not required when building a shared library. */ public static void main(String[] argv) {} From b09abff543f0f92b1a52088f588fc430982caabf Mon Sep 17 00:00:00 2001 From: Kushal Pisavadia Date: Wed, 7 May 2025 11:41:38 +0100 Subject: [PATCH 07/14] Add `@SuppressWarnings("unused")` annotation to `LibPkl` class --- libpkl/src/main/java/org/pkl/libpkl/LibPkl.java | 1 + 1 file changed, 1 insertion(+) diff --git a/libpkl/src/main/java/org/pkl/libpkl/LibPkl.java b/libpkl/src/main/java/org/pkl/libpkl/LibPkl.java index e2c95909b..c4d1c43d5 100644 --- a/libpkl/src/main/java/org/pkl/libpkl/LibPkl.java +++ b/libpkl/src/main/java/org/pkl/libpkl/LibPkl.java @@ -27,6 +27,7 @@ import org.pkl.core.messaging.ProtocolException; import org.pkl.server.Server; +@SuppressWarnings("unused") public class LibPkl { public interface MessageCallbackFunctionPointer extends CFunctionPointer { @InvokeCFunctionPointer(transition = Transition.TO_NATIVE) From 6102e4582614a9d510d9e215ece1423ca93196b5 Mon Sep 17 00:00:00 2001 From: Kushal Pisavadia Date: Wed, 7 May 2025 11:42:14 +0100 Subject: [PATCH 08/14] Move `LibPkl` constructor between fields and methods --- libpkl/src/main/java/org/pkl/libpkl/LibPkl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libpkl/src/main/java/org/pkl/libpkl/LibPkl.java b/libpkl/src/main/java/org/pkl/libpkl/LibPkl.java index c4d1c43d5..0c650ab5a 100644 --- a/libpkl/src/main/java/org/pkl/libpkl/LibPkl.java +++ b/libpkl/src/main/java/org/pkl/libpkl/LibPkl.java @@ -40,6 +40,8 @@ public interface MessageCallbackFunctionPointer extends CFunctionPointer { private static Server server; private static MessageCallbackFunctionPointer cb; + private LibPkl() {} + @CEntryPoint(name = "pkl_internal_init", builtin = CEntryPoint.Builtin.CREATE_ISOLATE) static native IsolateThread pklInternalInit(); @@ -90,6 +92,4 @@ public static void handleSendMessageToNative(byte[] bytes) { * argument is optional and not required when building a shared library. */ public static void main(String[] argv) {} - - private LibPkl() {} } From 19969648fbaffddf35a5035b8b34ca98440494bc Mon Sep 17 00:00:00 2001 From: Kushal Pisavadia Date: Wed, 7 May 2025 11:44:43 +0100 Subject: [PATCH 09/14] Add `package-info.java` for `org.pkl.libpkl` package --- libpkl/src/main/java/org/pkl/libpkl/package-info.java | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 libpkl/src/main/java/org/pkl/libpkl/package-info.java diff --git a/libpkl/src/main/java/org/pkl/libpkl/package-info.java b/libpkl/src/main/java/org/pkl/libpkl/package-info.java new file mode 100644 index 000000000..fb5f09e39 --- /dev/null +++ b/libpkl/src/main/java/org/pkl/libpkl/package-info.java @@ -0,0 +1,4 @@ +@NonnullByDefault +package org.pkl.libpkl; + +import org.pkl.core.util.NonnullByDefault; From 30c1a4639ea30bc9c728579f84e90435e7423121 Mon Sep 17 00:00:00 2001 From: Kushal Pisavadia Date: Wed, 7 May 2025 13:55:31 +0100 Subject: [PATCH 10/14] Propagate param `handlerContext` so users can trace messages This allows a user to provide a pointer, and have it be passed through and back to their `PklMessageResponseHandler` handler for them track as part of their own bookkeeping. --- libpkl/src/main/c/pkl.c | 4 ++-- libpkl/src/main/c/pkl.h | 4 ++-- libpkl/src/main/java/org/pkl/libpkl/LibPkl.java | 14 +++++++++----- .../main/java/org/pkl/libpkl/NativeTransport.java | 14 +++++++++----- .../test/kotlin/org/pkl/libpkl/JNATestClient.kt | 5 +++-- .../test/kotlin/org/pkl/libpkl/LibPklLibrary.kt | 4 ++-- 6 files changed, 27 insertions(+), 18 deletions(-) diff --git a/libpkl/src/main/c/pkl.c b/libpkl/src/main/c/pkl.c index ed038dcf5..60c6dddaf 100644 --- a/libpkl/src/main/c/pkl.c +++ b/libpkl/src/main/c/pkl.c @@ -38,12 +38,12 @@ int pkl_init(PklMessageResponseHandler handler) { }; -int pkl_send_message(int length, char* message) { +int pkl_send_message(int length, char *message, void *handlerContext) { if (pthread_mutex_lock(&graal_mutex) != 0) { return -1; } - pkl_internal_send_message(isolatethread, length, message); + pkl_internal_send_message(isolatethread, length, message, handlerContext); pthread_mutex_unlock(&graal_mutex); return 0; diff --git a/libpkl/src/main/c/pkl.h b/libpkl/src/main/c/pkl.h index 0e4a9deef..f90a42d41 100644 --- a/libpkl/src/main/c/pkl.h +++ b/libpkl/src/main/c/pkl.h @@ -3,7 +3,7 @@ * * The resulting messages from `pkl_send` will be sent to this handler using a callback style. */ -typedef void (*PklMessageResponseHandler)(int length, char *message); +typedef void (*PklMessageResponseHandler)(int length, char *message, void *handlerContext); /** * @brief Initialises and allocates a Pkl executor. @@ -19,7 +19,7 @@ int pkl_init(PklMessageResponseHandler handler); * @return -1 on failure. * @return 0 on success. */ -int pkl_send_message(int length, char *message); +int pkl_send_message(int length, char *message, void *handlerContext); /** * @brief Cleans up any resources that were created as part of the `pkl_init` process. diff --git a/libpkl/src/main/java/org/pkl/libpkl/LibPkl.java b/libpkl/src/main/java/org/pkl/libpkl/LibPkl.java index 0c650ab5a..848ecbf74 100644 --- a/libpkl/src/main/java/org/pkl/libpkl/LibPkl.java +++ b/libpkl/src/main/java/org/pkl/libpkl/LibPkl.java @@ -23,6 +23,8 @@ import org.graalvm.nativeimage.c.function.CFunctionPointer; import org.graalvm.nativeimage.c.function.InvokeCFunctionPointer; import org.graalvm.nativeimage.c.type.CCharPointer; +import org.graalvm.nativeimage.c.type.VoidPointer; +import org.graalvm.word.WordFactory; import org.pkl.core.messaging.MessageTransports.Logger; import org.pkl.core.messaging.ProtocolException; import org.pkl.server.Server; @@ -31,7 +33,7 @@ public class LibPkl { public interface MessageCallbackFunctionPointer extends CFunctionPointer { @InvokeCFunctionPointer(transition = Transition.TO_NATIVE) - void invoke(int length, CCharPointer msg); + void invoke(int length, CCharPointer msg, VoidPointer handlerContext); } private static final Logger logger = new LibPklLogger(); @@ -46,10 +48,11 @@ private LibPkl() {} static native IsolateThread pklInternalInit(); @CEntryPoint(name = "pkl_internal_send_message") - public static void pklInternalSendMessage(IsolateThread thread, int length, CCharPointer ptr) + public static void pklInternalSendMessage( + IsolateThread thread, int length, CCharPointer ptr, VoidPointer handlerContext) throws ProtocolException, IOException { logger.log("Got message from native"); - transport.sendMessage(length, ptr); + transport.sendMessage(length, ptr, handlerContext); } @CEntryPoint(name = "pkl_internal_register_response_handler") @@ -73,9 +76,10 @@ public static void pklInternalServerStop(IsolateThread thread) { server.close(); } - public static void handleSendMessageToNative(byte[] bytes) { + public static void handleSendMessageToNative(byte[] bytes, Object handlerContext) { try (var pin = PinnedObject.create(bytes)) { - cb.invoke(bytes.length, pin.addressOfArrayElement(0)); + // TODO: Propagate `handlerContext` through instead of `nullPointer`. + cb.invoke(bytes.length, pin.addressOfArrayElement(0), WordFactory.nullPointer()); } } diff --git a/libpkl/src/main/java/org/pkl/libpkl/NativeTransport.java b/libpkl/src/main/java/org/pkl/libpkl/NativeTransport.java index 400cc0117..d34162f06 100644 --- a/libpkl/src/main/java/org/pkl/libpkl/NativeTransport.java +++ b/libpkl/src/main/java/org/pkl/libpkl/NativeTransport.java @@ -17,8 +17,9 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.util.function.Consumer; +import java.util.function.BiConsumer; import org.graalvm.nativeimage.c.type.CCharPointer; +import org.graalvm.nativeimage.c.type.VoidPointer; import org.msgpack.core.MessagePack; import org.pkl.core.messaging.Message; import org.pkl.core.messaging.MessageTransports.AbstractMessageTransport; @@ -28,9 +29,9 @@ import org.pkl.server.ServerMessagePackEncoder; public class NativeTransport extends AbstractMessageTransport { - private final Consumer sendMessageToNative; + private final BiConsumer sendMessageToNative; - protected NativeTransport(Logger logger, Consumer sendMessageToNative) { + protected NativeTransport(Logger logger, BiConsumer sendMessageToNative) { super(logger); this.sendMessageToNative = sendMessageToNative; } @@ -47,11 +48,14 @@ protected void doSend(Message message) throws IOException, ProtocolException { var packer = MessagePack.newDefaultPacker(os)) { var encoder = new ServerMessagePackEncoder(packer); encoder.encode(message); - sendMessageToNative.accept(os.toByteArray()); + // TODO: Propagate `handlerContext` through to `sendMessageToNative` + sendMessageToNative.accept(os.toByteArray(), null); } } - public void sendMessage(int length, CCharPointer ptr) throws IOException, ProtocolException { + // TODO: Propagate `handlerContext` through to `sendMessageToNative` + public void sendMessage(int length, CCharPointer ptr, VoidPointer handlerContext) + throws IOException, ProtocolException { try (var is = new NativeInputStream(length, ptr); var unpacker = MessagePack.newDefaultUnpacker(is)) { var message = new ServerMessagePackDecoder(unpacker).decode(); diff --git a/libpkl/src/test/kotlin/org/pkl/libpkl/JNATestClient.kt b/libpkl/src/test/kotlin/org/pkl/libpkl/JNATestClient.kt index 8e7737137..8ac705adf 100644 --- a/libpkl/src/test/kotlin/org/pkl/libpkl/JNATestClient.kt +++ b/libpkl/src/test/kotlin/org/pkl/libpkl/JNATestClient.kt @@ -36,7 +36,7 @@ import org.pkl.server.ServerMessagePackEncoder class JNATestClient : LibPklLibrary.PklMessageResponseHandler, Iterable, AutoCloseable { val incoming: BlockingQueue = ArrayBlockingQueue(10) - override fun invoke(length: Int, message: Pointer?) { + override fun invoke(length: Int, message: Pointer?, handlerContext: Pointer?) { when { message != null && length > 0 -> { val receivedBytes: ByteArray = message.getByteArray(0, length) @@ -53,7 +53,8 @@ class JNATestClient : LibPklLibrary.PklMessageResponseHandler, Iterable = incoming.iterator() fun send(message: Message): Int = - encode(message).let { LibPklLibrary.INSTANCE.pkl_send_message(it.size, it) } + // TODO: Propagate `handlerContext` through, and validate it. + encode(message).let { LibPklLibrary.INSTANCE.pkl_send_message(it.size, it, null) } inline fun receive(): T { val message = incoming.take() diff --git a/libpkl/src/test/kotlin/org/pkl/libpkl/LibPklLibrary.kt b/libpkl/src/test/kotlin/org/pkl/libpkl/LibPklLibrary.kt index a3e417ad7..e8c1e9c22 100644 --- a/libpkl/src/test/kotlin/org/pkl/libpkl/LibPklLibrary.kt +++ b/libpkl/src/test/kotlin/org/pkl/libpkl/LibPklLibrary.kt @@ -26,12 +26,12 @@ interface LibPklLibrary : Library { } interface PklMessageResponseHandler : Callback { - fun invoke(length: Int, message: Pointer?) + fun invoke(length: Int, message: Pointer?, handlerContext: Pointer?) } fun pkl_init(handler: PklMessageResponseHandler?): Int - fun pkl_send_message(length: Int, message: ByteArray?): Int + fun pkl_send_message(length: Int, message: ByteArray?, handlerContext: Pointer?): Int fun pkl_close(): Int } From ba5d361c95ea9b2030fb3d893707e0e81ff3d0b7 Mon Sep 17 00:00:00 2001 From: Kushal Pisavadia Date: Wed, 7 May 2025 15:03:46 +0100 Subject: [PATCH 11/14] Drop the OS and Arch name from the `native-image` shared library --- libpkl/libpkl.gradle.kts | 20 ++++++++++---------- libpkl/src/main/c/pkl.c | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/libpkl/libpkl.gradle.kts b/libpkl/libpkl.gradle.kts index bb2960cd2..540ce7353 100644 --- a/libpkl/libpkl.gradle.kts +++ b/libpkl/libpkl.gradle.kts @@ -58,7 +58,7 @@ dependencies { } executable { - name = "libpkl-internal" + name = "libpkl_internal" // TODO(kushal): Why is all of this necessary now? Can it be stripped back? javaName = "libpkl" @@ -85,10 +85,10 @@ private fun extension(osAndArch: String) = private fun nativeLibraryOutputFiles(osAndArch: String) = project.layout.buildDirectory.dir("libs/$osAndArch").map { outputDir -> // TODO(kushal): dashes/underscores for library files? C convention assumes underscores. - val libraryName = "libpkl-$osAndArch" + val libraryName = executableSpec.name val libraryOutputFiles = listOf( - "${libraryName}.${extension(osAndArch)}", + "lib${libraryName}.${extension(osAndArch)}", "${libraryName}_dynamic.h", "${libraryName}.h", @@ -125,7 +125,7 @@ private fun NativeImageBuild.setClasspath() { val macNativeLibraryAmd64 by tasks.registering(NativeImageBuild::class) { outputDir = project.layout.buildDirectory.dir("libs/macos-amd64") - imageName = executableSpec.name.map { "$it-macos-amd64" } + imageName = executableSpec.name mainClass = executableSpec.mainClass amd64() setClasspath() @@ -137,7 +137,7 @@ val macNativeLibraryAmd64 by val macNativeLibraryAarch64 by tasks.registering(NativeImageBuild::class) { outputDir = project.layout.buildDirectory.dir("libs/macos-aarch64") - imageName = executableSpec.name.map { "$it-macos-aarch64" } + imageName = executableSpec.name mainClass = executableSpec.mainClass aarch64() setClasspath() @@ -149,7 +149,7 @@ val macNativeLibraryAarch64 by val linuxNativeLibraryAmd64 by tasks.registering(NativeImageBuild::class) { outputDir = project.layout.buildDirectory.dir("libs/linux-amd64") - imageName = executableSpec.name.map { "$it-linux-amd64" } + imageName = executableSpec.name mainClass = executableSpec.mainClass amd64() setClasspath() @@ -161,7 +161,7 @@ val linuxNativeLibraryAmd64 by val linuxNativeLibraryAarch64 by tasks.registering(NativeImageBuild::class) { outputDir = project.layout.buildDirectory.dir("libs/linux-aarch64") - imageName = executableSpec.name.map { "$it-linux-aarch64" } + imageName = executableSpec.name mainClass = executableSpec.mainClass aarch64() setClasspath() @@ -180,7 +180,7 @@ val linuxNativeLibraryAarch64 by val alpineNativeLibraryAmd64 by tasks.registering(NativeImageBuild::class) { outputDir = project.layout.buildDirectory.dir("libs/alpine-linux-amd64") - imageName = executableSpec.name.map { "$it-alpine-linux-amd64" } + imageName = executableSpec.name mainClass = executableSpec.mainClass amd64() setClasspath() @@ -198,7 +198,7 @@ val alpineNativeLibraryAmd64 by val windowsNativeLibraryAmd64 by tasks.registering(NativeImageBuild::class) { outputDir = project.layout.buildDirectory.dir("libs/windows-amd64") - imageName = executableSpec.name.map { "$it-windows-amd64" } + imageName = executableSpec.name mainClass = executableSpec.mainClass amd64() setClasspath() @@ -293,7 +293,7 @@ val macNativeFullLibraryAarch64 by "-I$projectDir/src/main/c", "-I$libraryOutputDir", "-L$libraryOutputDir", - "-lpkl-internal-macos-aarch64", + "-lpkl_internal", ) } diff --git a/libpkl/src/main/c/pkl.c b/libpkl/src/main/c/pkl.c index 60c6dddaf..6077c3fe9 100644 --- a/libpkl/src/main/c/pkl.c +++ b/libpkl/src/main/c/pkl.c @@ -3,7 +3,7 @@ #include #include -#include +#include #include From b1b1babbb6dafc9aaf7779856c40c7119b5c92e7 Mon Sep 17 00:00:00 2001 From: Kushal Pisavadia Date: Wed, 7 May 2025 15:14:24 +0100 Subject: [PATCH 12/14] TODO: Provide a meaningful error to user if `cb == null` --- libpkl/src/main/java/org/pkl/libpkl/LibPkl.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/libpkl/src/main/java/org/pkl/libpkl/LibPkl.java b/libpkl/src/main/java/org/pkl/libpkl/LibPkl.java index 848ecbf74..17e6b394f 100644 --- a/libpkl/src/main/java/org/pkl/libpkl/LibPkl.java +++ b/libpkl/src/main/java/org/pkl/libpkl/LibPkl.java @@ -78,8 +78,11 @@ public static void pklInternalServerStop(IsolateThread thread) { public static void handleSendMessageToNative(byte[] bytes, Object handlerContext) { try (var pin = PinnedObject.create(bytes)) { - // TODO: Propagate `handlerContext` through instead of `nullPointer`. - cb.invoke(bytes.length, pin.addressOfArrayElement(0), WordFactory.nullPointer()); + // TODO: Provide a meaningful error the user if they haven't run `pkl_init`. + if (cb != null) { + // TODO: Propagate `handlerContext` through instead of `nullPointer`. + cb.invoke(bytes.length, pin.addressOfArrayElement(0), WordFactory.nullPointer()); + } } } From 0fd8ab38bcccb1b3a3b982ee71fbba098fdafa68 Mon Sep 17 00:00:00 2001 From: Kushal Pisavadia Date: Tue, 27 May 2025 10:11:07 +0100 Subject: [PATCH 13/14] Log exceptions in `NativeTransport` --- .../main/java/org/pkl/libpkl/NativeTransport.java | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/libpkl/src/main/java/org/pkl/libpkl/NativeTransport.java b/libpkl/src/main/java/org/pkl/libpkl/NativeTransport.java index d34162f06..e40b0bcd4 100644 --- a/libpkl/src/main/java/org/pkl/libpkl/NativeTransport.java +++ b/libpkl/src/main/java/org/pkl/libpkl/NativeTransport.java @@ -30,37 +30,44 @@ public class NativeTransport extends AbstractMessageTransport { private final BiConsumer sendMessageToNative; + private final Logger logger; protected NativeTransport(Logger logger, BiConsumer sendMessageToNative) { super(logger); + this.logger = logger; this.sendMessageToNative = sendMessageToNative; } @Override - protected void doStart() throws ProtocolException, IOException {} + protected void doStart() {} @Override protected void doClose() {} @Override - protected void doSend(Message message) throws IOException, ProtocolException { + protected void doSend(Message message) { try (var os = new ByteArrayOutputStream(); var packer = MessagePack.newDefaultPacker(os)) { var encoder = new ServerMessagePackEncoder(packer); encoder.encode(message); // TODO: Propagate `handlerContext` through to `sendMessageToNative` sendMessageToNative.accept(os.toByteArray(), null); + } catch (IOException | ProtocolException e) { + // TODO: Test that this error message is visible. + logger.log(e.getMessage()); } } // TODO: Propagate `handlerContext` through to `sendMessageToNative` - public void sendMessage(int length, CCharPointer ptr, VoidPointer handlerContext) - throws IOException, ProtocolException { + public void sendMessage(int length, CCharPointer ptr, VoidPointer handlerContext) { try (var is = new NativeInputStream(length, ptr); var unpacker = MessagePack.newDefaultUnpacker(is)) { var message = new ServerMessagePackDecoder(unpacker).decode(); assert message != null; accept(message); + } catch (IOException | ProtocolException e) { + // TODO: Test that this error message is visible. + logger.log(e.getMessage()); } } } From e4ca22c490e1a2d67aef6b0d2b663ae674046cde Mon Sep 17 00:00:00 2001 From: Dan Chao Date: Wed, 28 May 2025 14:11:48 -0700 Subject: [PATCH 14/14] Address PR comments --- gradle/libs.versions.toml | 1 + libpkl/libpkl.gradle.kts | 83 +++++++++---------- libpkl/src/main/c/pkl.c | 26 ++++-- libpkl/src/main/c/pkl.h | 66 ++++++++++----- .../src/main/java/org/pkl/libpkl/LibPkl.java | 22 ++--- .../org/pkl/libpkl/NativeInputStream.java | 17 ++-- .../java/org/pkl/libpkl/NativeTransport.java | 13 ++- .../kotlin/org/pkl/libpkl/JNATestClient.kt | 19 ++--- .../kotlin/org/pkl/libpkl/LibPklLibrary.kt | 6 +- .../pkl/libpkl/{JNATest.kt => NativeTest.kt} | 19 +++-- 10 files changed, 152 insertions(+), 120 deletions(-) rename libpkl/src/test/kotlin/org/pkl/libpkl/{JNATest.kt => NativeTest.kt} (97%) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 79dc64bc1..750ba6e8a 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -2,6 +2,7 @@ antlr = "4.+" assertj = "3.+" checksumPlugin = "1.4.0" +clangfmt = "10.0.1" clikt = "5.+" commonMark = "0.+" downloadTaskPlugin = "5.6.0" diff --git a/libpkl/libpkl.gradle.kts b/libpkl/libpkl.gradle.kts index 540ce7353..df1a8eebb 100644 --- a/libpkl/libpkl.gradle.kts +++ b/libpkl/libpkl.gradle.kts @@ -1,29 +1,27 @@ /* - * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +* Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* https://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +@file:Suppress("unused") + plugins { pklAllProjects pklGraalVm pklJavaLibrary - pklJavaExecutable pklNativeLifecycle } -// assumes that `pklJavaExecutable` is also applied -val executableSpec = project.extensions.getByType() - val stagedMacAmd64NativeLibrary: Configuration by configurations.creating val stagedMacAarch64NativeLibrary: Configuration by configurations.creating val stagedLinuxAmd64NativeLibrary: Configuration by configurations.creating @@ -57,18 +55,6 @@ dependencies { stagedWindowsAmd64NativeLibrary(sharedLibrary("windows-amd64.exe")) } -executable { - name = "libpkl_internal" - - // TODO(kushal): Why is all of this necessary now? Can it be stripped back? - javaName = "libpkl" - documentationName = "Pkl Native Library" - publicationName = "libpkl" - javaPublicationName = "libpkl" - mainClass = "org.pkl.libpkl.LibPkl" - website = "TODO" -} - private fun extension(osAndArch: String) = when (osAndArch.split("-").dropWhile { it == "alpine" }.first()) { "linux" -> "so" @@ -85,7 +71,7 @@ private fun extension(osAndArch: String) = private fun nativeLibraryOutputFiles(osAndArch: String) = project.layout.buildDirectory.dir("libs/$osAndArch").map { outputDir -> // TODO(kushal): dashes/underscores for library files? C convention assumes underscores. - val libraryName = executableSpec.name + val libraryName = "libpkl_internal" val libraryOutputFiles = listOf( "lib${libraryName}.${extension(osAndArch)}", @@ -125,8 +111,8 @@ private fun NativeImageBuild.setClasspath() { val macNativeLibraryAmd64 by tasks.registering(NativeImageBuild::class) { outputDir = project.layout.buildDirectory.dir("libs/macos-amd64") - imageName = executableSpec.name - mainClass = executableSpec.mainClass + imageName = "libpkl_internal" + mainClass = "org.pkl.libpkl.LibPkl" amd64() setClasspath() extraNativeImageArgs = listOf("--shared") @@ -137,8 +123,8 @@ val macNativeLibraryAmd64 by val macNativeLibraryAarch64 by tasks.registering(NativeImageBuild::class) { outputDir = project.layout.buildDirectory.dir("libs/macos-aarch64") - imageName = executableSpec.name - mainClass = executableSpec.mainClass + imageName = "libpkl_internal" + mainClass = "org.pkl.libpkl.LibPkl" aarch64() setClasspath() extraNativeImageArgs = listOf("--shared") @@ -149,8 +135,8 @@ val macNativeLibraryAarch64 by val linuxNativeLibraryAmd64 by tasks.registering(NativeImageBuild::class) { outputDir = project.layout.buildDirectory.dir("libs/linux-amd64") - imageName = executableSpec.name - mainClass = executableSpec.mainClass + imageName = "libpkl_internal" + mainClass = "org.pkl.libpkl.LibPkl" amd64() setClasspath() extraNativeImageArgs = listOf("--shared") @@ -161,8 +147,8 @@ val linuxNativeLibraryAmd64 by val linuxNativeLibraryAarch64 by tasks.registering(NativeImageBuild::class) { outputDir = project.layout.buildDirectory.dir("libs/linux-aarch64") - imageName = executableSpec.name - mainClass = executableSpec.mainClass + imageName = "libpkl_internal" + mainClass = "org.pkl.libpkl.LibPkl" aarch64() setClasspath() @@ -180,8 +166,8 @@ val linuxNativeLibraryAarch64 by val alpineNativeLibraryAmd64 by tasks.registering(NativeImageBuild::class) { outputDir = project.layout.buildDirectory.dir("libs/alpine-linux-amd64") - imageName = executableSpec.name - mainClass = executableSpec.mainClass + imageName = "libpkl_internal" + mainClass = "org.pkl.libpkl.LibPkl" amd64() setClasspath() @@ -198,8 +184,8 @@ val alpineNativeLibraryAmd64 by val windowsNativeLibraryAmd64 by tasks.registering(NativeImageBuild::class) { outputDir = project.layout.buildDirectory.dir("libs/windows-amd64") - imageName = executableSpec.name - mainClass = executableSpec.mainClass + imageName = "libpkl_internal" + mainClass = "org.pkl.libpkl.LibPkl" amd64() setClasspath() extraNativeImageArgs = listOf("--shared", "-Dfile.encoding=UTF-8") @@ -317,3 +303,14 @@ tasks.withType { useJUnitPlatform() } + +private val licenseHeaderFile by lazy { + rootProject.file("buildSrc/src/main/resources/license-header.star-block.txt") +} + +spotless { + cpp { + licenseHeaderFile(licenseHeaderFile, "// ") + target("src/main/c/*.c", "src/main/c/*.h") + } +} diff --git a/libpkl/src/main/c/pkl.c b/libpkl/src/main/c/pkl.c index 6077c3fe9..b27622d7c 100644 --- a/libpkl/src/main/c/pkl.c +++ b/libpkl/src/main/c/pkl.c @@ -1,3 +1,20 @@ +/* + * Copyright © 2025 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +// pkl.c + #include #include #include @@ -14,7 +31,7 @@ pthread_mutex_t graal_mutex; graal_isolatethread_t *isolatethread = NULL; -int pkl_init(PklMessageResponseHandler handler) { +int pkl_init(PklMessageResponseHandler handler, void *payload) { if (isolatethread != NULL) { perror("pkl_init: isolatethread is already initialised"); return -1; @@ -30,20 +47,19 @@ int pkl_init(PklMessageResponseHandler handler) { } isolatethread = pkl_internal_init(); - pkl_internal_register_response_handler(isolatethread, handler); + pkl_internal_register_response_handler(isolatethread, handler, payload); pkl_internal_server_start(isolatethread); pthread_mutex_unlock(&graal_mutex); return 0; }; - -int pkl_send_message(int length, char *message, void *handlerContext) { +int pkl_send_message(int length, char *message) { if (pthread_mutex_lock(&graal_mutex) != 0) { return -1; } - pkl_internal_send_message(isolatethread, length, message, handlerContext); + pkl_internal_send_message(isolatethread, length, message); pthread_mutex_unlock(&graal_mutex); return 0; diff --git a/libpkl/src/main/c/pkl.h b/libpkl/src/main/c/pkl.h index f90a42d41..4c2df64c6 100644 --- a/libpkl/src/main/c/pkl.h +++ b/libpkl/src/main/c/pkl.h @@ -1,30 +1,52 @@ +/* + * Copyright © 2025 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +// pkl.h + /** -* @brief The Pkl Message Response Handler that a user should implement. -* -* The resulting messages from `pkl_send` will be sent to this handler using a callback style. -*/ -typedef void (*PklMessageResponseHandler)(int length, char *message, void *handlerContext); + * The callback that gets called when a message is received from Pkl. + * + * @param length The length the message bytes + * @param message The message itself + * @param payload User-defined data passed to pkl_init. + */ +typedef void (*PklMessageResponseHandler)(int length, char *message, void *payload); /** -* @brief Initialises and allocates a Pkl executor. -* -* @return -1 on failure. -* @return 0 on success. -*/ -int pkl_init(PklMessageResponseHandler handler); + * Initialises and allocates a Pkl executor. + * + * @param handler The callback that gets called when a message is received from Pkl. + * @param payload User-defined data that gets passed to handler. + * + * @return -1 on failure, 0 on success. + */ +int pkl_init(PklMessageResponseHandler handler, void *payload); /** -* @brief Send a message to Pkl, providing the length and a pointer to the first byte. -* -* @return -1 on failure. -* @return 0 on success. -*/ -int pkl_send_message(int length, char *message, void *handlerContext); + * Send a message to Pkl, providing the length and a pointer to the first byte. + * + * @param length The length of the message, in bytes. + * @param message The message to send to Pkl. + * + * @return -1 on failure, 0 on success. + */ +int pkl_send_message(int length, char *message); /** -* @brief Cleans up any resources that were created as part of the `pkl_init` process. -* -* @return -1 on failure. -* @return 0 on success. -*/ + * Cleans up any resources that were created as part of the `pkl_init` process. + * + * @return -1 on failure, 0 on success. + */ int pkl_close(); diff --git a/libpkl/src/main/java/org/pkl/libpkl/LibPkl.java b/libpkl/src/main/java/org/pkl/libpkl/LibPkl.java index 17e6b394f..409afce75 100644 --- a/libpkl/src/main/java/org/pkl/libpkl/LibPkl.java +++ b/libpkl/src/main/java/org/pkl/libpkl/LibPkl.java @@ -15,7 +15,6 @@ */ package org.pkl.libpkl; -import java.io.IOException; import org.graalvm.nativeimage.IsolateThread; import org.graalvm.nativeimage.PinnedObject; import org.graalvm.nativeimage.c.function.CEntryPoint; @@ -24,16 +23,14 @@ import org.graalvm.nativeimage.c.function.InvokeCFunctionPointer; import org.graalvm.nativeimage.c.type.CCharPointer; import org.graalvm.nativeimage.c.type.VoidPointer; -import org.graalvm.word.WordFactory; import org.pkl.core.messaging.MessageTransports.Logger; -import org.pkl.core.messaging.ProtocolException; import org.pkl.server.Server; @SuppressWarnings("unused") public class LibPkl { public interface MessageCallbackFunctionPointer extends CFunctionPointer { @InvokeCFunctionPointer(transition = Transition.TO_NATIVE) - void invoke(int length, CCharPointer msg, VoidPointer handlerContext); + void invoke(int length, CCharPointer msg, VoidPointer userData); } private static final Logger logger = new LibPklLogger(); @@ -41,6 +38,7 @@ public interface MessageCallbackFunctionPointer extends CFunctionPointer { new NativeTransport(logger, LibPkl::handleSendMessageToNative); private static Server server; private static MessageCallbackFunctionPointer cb; + private static VoidPointer userData; private LibPkl() {} @@ -48,18 +46,17 @@ private LibPkl() {} static native IsolateThread pklInternalInit(); @CEntryPoint(name = "pkl_internal_send_message") - public static void pklInternalSendMessage( - IsolateThread thread, int length, CCharPointer ptr, VoidPointer handlerContext) - throws ProtocolException, IOException { + public static void pklInternalSendMessage(IsolateThread thread, int length, CCharPointer ptr) { logger.log("Got message from native"); - transport.sendMessage(length, ptr, handlerContext); + transport.sendMessage(length, ptr); } @CEntryPoint(name = "pkl_internal_register_response_handler") public static void pklInternalRegisterResponseHandler( - IsolateThread thread, LibPkl.MessageCallbackFunctionPointer cb) { + IsolateThread thread, LibPkl.MessageCallbackFunctionPointer cb, VoidPointer userData) { logger.log("Got handler to call from Pkl"); LibPkl.cb = cb; + LibPkl.userData = userData; } @CEntryPoint(name = "pkl_internal_close", builtin = CEntryPoint.Builtin.TEAR_DOWN_ISOLATE) @@ -76,13 +73,10 @@ public static void pklInternalServerStop(IsolateThread thread) { server.close(); } - public static void handleSendMessageToNative(byte[] bytes, Object handlerContext) { + public static void handleSendMessageToNative(byte[] bytes) { try (var pin = PinnedObject.create(bytes)) { // TODO: Provide a meaningful error the user if they haven't run `pkl_init`. - if (cb != null) { - // TODO: Propagate `handlerContext` through instead of `nullPointer`. - cb.invoke(bytes.length, pin.addressOfArrayElement(0), WordFactory.nullPointer()); - } + cb.invoke(bytes.length, pin.addressOfArrayElement(0), LibPkl.userData); } } diff --git a/libpkl/src/main/java/org/pkl/libpkl/NativeInputStream.java b/libpkl/src/main/java/org/pkl/libpkl/NativeInputStream.java index ca343575b..5f58ee592 100644 --- a/libpkl/src/main/java/org/pkl/libpkl/NativeInputStream.java +++ b/libpkl/src/main/java/org/pkl/libpkl/NativeInputStream.java @@ -23,21 +23,24 @@ public class NativeInputStream extends InputStream { private final int length; private final CCharPointer ptr; + public NativeInputStream(int length, CCharPointer ptr) { + super(); + this.length = length; + this.ptr = ptr; + } + @Override public int read() { + if (available() <= 0) { + return -1; + } var result = ptr.read(offset); offset++; - return result; + return result & 0xFF; } @Override public int available() { return length - offset; } - - public NativeInputStream(int length, CCharPointer ptr) { - super(); - this.length = length; - this.ptr = ptr; - } } diff --git a/libpkl/src/main/java/org/pkl/libpkl/NativeTransport.java b/libpkl/src/main/java/org/pkl/libpkl/NativeTransport.java index e40b0bcd4..4de65bac8 100644 --- a/libpkl/src/main/java/org/pkl/libpkl/NativeTransport.java +++ b/libpkl/src/main/java/org/pkl/libpkl/NativeTransport.java @@ -17,9 +17,8 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.util.function.BiConsumer; +import java.util.function.Consumer; import org.graalvm.nativeimage.c.type.CCharPointer; -import org.graalvm.nativeimage.c.type.VoidPointer; import org.msgpack.core.MessagePack; import org.pkl.core.messaging.Message; import org.pkl.core.messaging.MessageTransports.AbstractMessageTransport; @@ -29,10 +28,10 @@ import org.pkl.server.ServerMessagePackEncoder; public class NativeTransport extends AbstractMessageTransport { - private final BiConsumer sendMessageToNative; + private final Consumer sendMessageToNative; private final Logger logger; - protected NativeTransport(Logger logger, BiConsumer sendMessageToNative) { + protected NativeTransport(Logger logger, Consumer sendMessageToNative) { super(logger); this.logger = logger; this.sendMessageToNative = sendMessageToNative; @@ -50,16 +49,14 @@ protected void doSend(Message message) { var packer = MessagePack.newDefaultPacker(os)) { var encoder = new ServerMessagePackEncoder(packer); encoder.encode(message); - // TODO: Propagate `handlerContext` through to `sendMessageToNative` - sendMessageToNative.accept(os.toByteArray(), null); + sendMessageToNative.accept(os.toByteArray()); } catch (IOException | ProtocolException e) { // TODO: Test that this error message is visible. logger.log(e.getMessage()); } } - // TODO: Propagate `handlerContext` through to `sendMessageToNative` - public void sendMessage(int length, CCharPointer ptr, VoidPointer handlerContext) { + public void sendMessage(int length, CCharPointer ptr) { try (var is = new NativeInputStream(length, ptr); var unpacker = MessagePack.newDefaultUnpacker(is)) { var message = new ServerMessagePackDecoder(unpacker).decode(); diff --git a/libpkl/src/test/kotlin/org/pkl/libpkl/JNATestClient.kt b/libpkl/src/test/kotlin/org/pkl/libpkl/JNATestClient.kt index 8ac705adf..4167e9bfe 100644 --- a/libpkl/src/test/kotlin/org/pkl/libpkl/JNATestClient.kt +++ b/libpkl/src/test/kotlin/org/pkl/libpkl/JNATestClient.kt @@ -34,18 +34,13 @@ import org.pkl.server.ServerMessagePackDecoder import org.pkl.server.ServerMessagePackEncoder class JNATestClient : LibPklLibrary.PklMessageResponseHandler, Iterable, AutoCloseable { - val incoming: BlockingQueue = ArrayBlockingQueue(10) + val incoming: BlockingQueue = ArrayBlockingQueue(10) - override fun invoke(length: Int, message: Pointer?, handlerContext: Pointer?) { - when { - message != null && length > 0 -> { - val receivedBytes: ByteArray = message.getByteArray(0, length) - val message = decode(receivedBytes) - assertThat(message).isInstanceOf(Message::class.java) - incoming.add(message!!) - } - else -> incoming.add(null) - } + override fun invoke(length: Int, message: Pointer, userData: Pointer?) { + val receivedBytes: ByteArray = message.getByteArray(0, length) + val message = decode(receivedBytes) + assertThat(message).isInstanceOf(Message::class.java) + incoming.add(message!!) } override fun close() = incoming.clear() @@ -54,7 +49,7 @@ class JNATestClient : LibPklLibrary.PklMessageResponseHandler, Iterable receive(): T { val message = incoming.take() diff --git a/libpkl/src/test/kotlin/org/pkl/libpkl/LibPklLibrary.kt b/libpkl/src/test/kotlin/org/pkl/libpkl/LibPklLibrary.kt index e8c1e9c22..2c54ac666 100644 --- a/libpkl/src/test/kotlin/org/pkl/libpkl/LibPklLibrary.kt +++ b/libpkl/src/test/kotlin/org/pkl/libpkl/LibPklLibrary.kt @@ -26,12 +26,12 @@ interface LibPklLibrary : Library { } interface PklMessageResponseHandler : Callback { - fun invoke(length: Int, message: Pointer?, handlerContext: Pointer?) + fun invoke(length: Int, message: Pointer, userData: Pointer?) } - fun pkl_init(handler: PklMessageResponseHandler?): Int + fun pkl_init(handler: PklMessageResponseHandler, userData: Pointer?): Int - fun pkl_send_message(length: Int, message: ByteArray?, handlerContext: Pointer?): Int + fun pkl_send_message(length: Int, message: ByteArray): Int fun pkl_close(): Int } diff --git a/libpkl/src/test/kotlin/org/pkl/libpkl/JNATest.kt b/libpkl/src/test/kotlin/org/pkl/libpkl/NativeTest.kt similarity index 97% rename from libpkl/src/test/kotlin/org/pkl/libpkl/JNATest.kt rename to libpkl/src/test/kotlin/org/pkl/libpkl/NativeTest.kt index 707217f6d..557d38764 100644 --- a/libpkl/src/test/kotlin/org/pkl/libpkl/JNATest.kt +++ b/libpkl/src/test/kotlin/org/pkl/libpkl/NativeTest.kt @@ -15,6 +15,7 @@ */ package org.pkl.libpkl +import com.sun.jna.Pointer import java.net.URI import java.nio.file.Path import kotlin.io.path.createDirectories @@ -39,12 +40,20 @@ import org.pkl.core.messaging.Messages.ResourceReaderSpec import org.pkl.core.module.PathElement import org.pkl.server.* +// To run this test in IntelliJ, add +// `-Djna.library.path=$ProjectFileDir$/libpkl/build/libs/-` to the +// run configuration. +// +// You can modify the IntelliJ JUnit configuration template so that this flag gets added +// automatically. +// See https://www.jetbrains.com/help/idea/run-debug-configuration.html#templates for more details. /** - * This test cannot be run inside of IntelliJ. + * Binds to the native library using JNA. * - * It needs the `jna.library.path` property to be set, which is handled by `libpkl:test`. + * @see JNATestClient + * @see LibPklLibrary */ -class JNATest { +class NativeTest { companion object { lateinit var client: JNATestClient } @@ -52,7 +61,7 @@ class JNATest { @BeforeEach fun beforeEach() { client = JNATestClient() - assertThat(LibPklLibrary.INSTANCE.pkl_init(client)).isEqualTo(0) + assertThat(LibPklLibrary.INSTANCE.pkl_init(client, Pointer.NULL)).isEqualTo(0) } @AfterEach @@ -66,8 +75,6 @@ class JNATest { val evaluatorId = client.sendCreateEvaluatorRequest() client.send(EvaluateRequest(1, evaluatorId, URI("repl:text"), """foo = 1""", null)) - assertThat(client).hasSize(1) - val response = client.receive() assertThat(response.evaluatorId).isEqualTo(evaluatorId)