Skip to content

[pixels-daemon] gRPC dependency issues #1271

New issue
New issue
Open
Open
[pixels-daemon] gRPC dependency issues#1271
@gengdy1545

Description

@gengdy1545
gengdy1545
opened on Jan 6, 2026

pixels-daemon.log:

2026-01-06 05:47:34,805 [io.pixelsdb.pixels.daemon.Daemon]-[INFO] starting daemon thread...
2026-01-06 05:47:35,812 [io.pixelsdb.pixels.common.utils.EtcdUtil]-[INFO] Using etcd host: localhost
2026-01-06 05:47:36,343 [io.pixelsdb.pixels.daemon.heartbeat.HeartbeatCoordinator]-[INFO] Heartbeat coordinator on ip-172-31-1-226 is initialized
2026-01-06 05:47:36,343 [io.pixelsdb.pixels.daemon.heartbeat.HeartbeatCoordinator]-[INFO] Starting heartbeat coordinator
2026-01-06 05:47:36,344 [io.pixelsdb.pixels.daemon.heartbeat.HeartbeatCoordinator]-[INFO] Heartbeat coordinator is running
2026-01-06 05:47:36,976 [io.pixelsdb.pixels.daemon.node.NodeServiceImpl]-[INFO] Initial hash ring loaded with 0 physical nodes and 128 virtual nodes.
2026-01-06 05:47:40,412 [io.pixelsdb.pixels.daemon.Daemon]-[INFO] starting daemon thread...
2026-01-06 05:47:41,418 [io.pixelsdb.pixels.common.utils.EtcdUtil]-[INFO] Using etcd host: localhost
2026-01-06 05:47:41,795 [io.pixelsdb.pixels.common.utils.EtcdUtil]-[ERROR] error when get key-values by prefix.
java.util.concurrent.ExecutionException: io.grpc.StatusRuntimeException: INTERNAL: Panic! This is a bug!
	at java.util.concurrent.CompletableFuture.wrapInExecutionException(CompletableFuture.java:345) ~[?:?]
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:440) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2117) ~[?:?]
	at io.pixelsdb.pixels.common.utils.EtcdUtil.getKeyValueByPrefix(EtcdUtil.java:160) ~[pixels-daemon-0.2.0-SNAPSHOT-full.jar:?]
	at io.pixelsdb.pixels.daemon.heartbeat.HeartbeatWorker.initialize(HeartbeatWorker.java:88) ~[pixels-daemon-0.2.0-SNAPSHOT-full.jar:?]
	at io.pixelsdb.pixels.daemon.heartbeat.HeartbeatWorker.<init>(HeartbeatWorker.java:73) ~[pixels-daemon-0.2.0-SNAPSHOT-full.jar:?]
	at io.pixelsdb.pixels.daemon.DaemonMain.main(DaemonMain.java:146) ~[pixels-daemon-0.2.0-SNAPSHOT-full.jar:?]
	at io.pixelsdb.pixels.daemon.PixelsWorker.main(PixelsWorker.java:30) ~[pixels-daemon-0.2.0-SNAPSHOT-full.jar:?]
Caused by: io.grpc.StatusRuntimeException: INTERNAL: Panic! This is a bug!
	at io.grpc.Status.asRuntimeException(Status.java:535) ~[ubuntu/:?]
	at io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onClose(ClientCalls.java:487) ~[ubuntu/:?]
	at io.grpc.internal.DelayedClientCall$DelayedListener$3.run(DelayedClientCall.java:470) ~[ubuntu/:?]
	at io.grpc.internal.DelayedClientCall$DelayedListener.delayOrExecute(DelayedClientCall.java:434) ~[ubuntu/:?]
	at io.grpc.internal.DelayedClientCall$DelayedListener.onClose(DelayedClientCall.java:467) ~[ubuntu/:?]
	at io.grpc.internal.ClientCallImpl.closeObserver(ClientCallImpl.java:563) ~[ubuntu/:?]
	at io.grpc.internal.ClientCallImpl.access$300(ClientCallImpl.java:70) ~[ubuntu/:?]
	at io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1StreamClosed.runInternal(ClientCallImpl.java:744) ~[ubuntu/:?]
	at io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1StreamClosed.runInContext(ClientCallImpl.java:723) ~[ubuntu/:?]
	at io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) ~[ubuntu/:?]
	at io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:133) ~[ubuntu/:?]
	at io.vertx.grpc.VertxChannelBuilder.lambda$build$1(VertxChannelBuilder.java:307) ~[pixels-daemon-0.2.0-SNAPSHOT-full.jar:?]
	at io.grpc.internal.SerializingExecutor.schedule(SerializingExecutor.java:102) ~[ubuntu/:?]
	at io.grpc.internal.SerializingExecutor.execute(SerializingExecutor.java:95) ~[ubuntu/:?]
	at io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl.closedInternal(ClientCallImpl.java:752) ~[ubuntu/:?]
	at io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl.closed(ClientCallImpl.java:688) ~[ubuntu/:?]
	at io.grpc.internal.RetriableStream$Sublistener$5.run(RetriableStream.java:986) ~[ubuntu/:?]
	at io.grpc.SynchronizationContext.drain(SynchronizationContext.java:95) ~[ubuntu/:?]
	at io.grpc.SynchronizationContext.execute(SynchronizationContext.java:127) ~[ubuntu/:?]
	at io.grpc.internal.RetriableStream$Sublistener.closed(RetriableStream.java:981) ~[ubuntu/:?]
	at io.grpc.internal.DelayedStream$DelayedStreamListener$4.run(DelayedStream.java:510) ~[ubuntu/:?]
	at io.grpc.internal.DelayedStream$DelayedStreamListener.drainPendingCallbacks(DelayedStream.java:535) ~[ubuntu/:?]
	at io.grpc.internal.DelayedStream.drainPendingCalls(DelayedStream.java:187) ~[ubuntu/:?]
	at io.grpc.internal.DelayedStream.access$100(DelayedStream.java:44) ~[ubuntu/:?]
	at io.grpc.internal.DelayedStream$4.run(DelayedStream.java:148) ~[ubuntu/:?]
	at io.vertx.grpc.VertxChannelBuilder.lambda$build$1(VertxChannelBuilder.java:307) ~[pixels-daemon-0.2.0-SNAPSHOT-full.jar:?]
	at io.grpc.internal.DelayedClientTransport.reprocess(DelayedClientTransport.java:311) ~[ubuntu/:?]
	at io.grpc.internal.ManagedChannelImpl.updateSubchannelPicker(ManagedChannelImpl.java:896) ~[ubuntu/:?]
	at io.grpc.internal.ManagedChannelImpl.panic(ManagedChannelImpl.java:882) ~[ubuntu/:?]
	at io.grpc.internal.ManagedChannelImpl$2.uncaughtException(ManagedChannelImpl.java:186) ~[ubuntu/:?]
	at io.grpc.SynchronizationContext.drain(SynchronizationContext.java:97) ~[ubuntu/:?]
	at io.grpc.SynchronizationContext.execute(SynchronizationContext.java:127) ~[ubuntu/:?]
	at io.grpc.internal.ManagedChannelImpl$NameResolverListener.onResult(ManagedChannelImpl.java:1870) ~[ubuntu/:?]
	at io.grpc.NameResolver$Listener2.onAddresses(NameResolver.java:215) ~[ubuntu/:?]
	at io.etcd.jetcd.resolver.AbstractNameResolver.doResolve(AbstractNameResolver.java:124) ~[pixels-daemon-0.2.0-SNAPSHOT-full.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) ~[?:?]
	at java.lang.Thread.run(Thread.java:1575) ~[?:?]
Caused by: java.lang.NoSuchMethodError: 'boolean io.grpc.internal.GrpcUtil.getFlag(java.lang.String, boolean)'
	at io.grpc.netty.NettyClientHandler.<clinit>(NettyClientHandler.java:101) ~[pixels-daemon-0.2.0-SNAPSHOT-full.jar:?]
	at io.grpc.netty.NettyClientTransport.start(NettyClientTransport.java:238) ~[pixels-daemon-0.2.0-SNAPSHOT-full.jar:?]
	at io.grpc.internal.ForwardingConnectionClientTransport.start(ForwardingConnectionClientTransport.java:34) ~[ubuntu/:?]
	at io.grpc.internal.ForwardingConnectionClientTransport.start(ForwardingConnectionClientTransport.java:34) ~[ubuntu/:?]
	at io.grpc.internal.InternalSubchannel.startNewTransport(InternalSubchannel.java:259) ~[ubuntu/:?]
	at io.grpc.internal.InternalSubchannel.access$400(InternalSubchannel.java:66) ~[ubuntu/:?]
	at io.grpc.internal.InternalSubchannel$2.run(InternalSubchannel.java:201) ~[ubuntu/:?]
	at io.grpc.SynchronizationContext.drain(SynchronizationContext.java:95) ~[ubuntu/:?]
	at io.grpc.SynchronizationContext.execute(SynchronizationContext.java:127) ~[ubuntu/:?]
	at io.grpc.internal.ManagedChannelImpl$NameResolverListener.onResult(ManagedChannelImpl.java:1870) ~[ubuntu/:?]
	at io.grpc.NameResolver$Listener2.onAddresses(NameResolver.java:215) ~[ubuntu/:?]
	at io.etcd.jetcd.resolver.AbstractNameResolver.doResolve(AbstractNameResolver.java:124) ~[pixels-daemon-0.2.0-SNAPSHOT-full.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) ~[?:?]
	at java.lang.Thread.run(Thread.java:1575) ~[?:?]
2026-01-06 05:47:41,804 [io.pixelsdb.pixels.daemon.heartbeat.HeartbeatWorker]-[ERROR] No heartbeat coordinator found, exit...
2026-01-06 05:47:41,804 [io.pixelsdb.pixels.daemon.heartbeat.HeartbeatWorker]-[INFO] Starting heartbeat worker
2026-01-06 05:47:41,804 [io.pixelsdb.pixels.daemon.heartbeat.HeartbeatWorker]-[ERROR] Heartbeat worker initialization failed, exit now...
2026-01-06 05:47:45,726 [io.pixelsdb.pixels.daemon.Daemon]-[INFO] starting daemon thread...
2026-01-06 05:47:46,732 [io.pixelsdb.pixels.common.utils.EtcdUtil]-[INFO] Using etcd host: localhost
2026-01-06 05:47:47,126 [io.pixelsdb.pixels.common.utils.EtcdUtil]-[ERROR] error when get key-values by prefix.
java.util.concurrent.ExecutionException: io.grpc.StatusRuntimeException: INTERNAL: Panic! This is a bug!
	at java.util.concurrent.CompletableFuture.wrapInExecutionException(CompletableFuture.java:345) ~[?:?]
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:440) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2117) ~[?:?]
	at io.pixelsdb.pixels.common.utils.EtcdUtil.getKeyValueByPrefix(EtcdUtil.java:160) ~[pixels-daemon-0.2.0-SNAPSHOT-full.jar:?]
	at io.pixelsdb.pixels.daemon.heartbeat.HeartbeatWorker.initialize(HeartbeatWorker.java:88) ~[pixels-daemon-0.2.0-SNAPSHOT-full.jar:?]
	at io.pixelsdb.pixels.daemon.heartbeat.HeartbeatWorker.<init>(HeartbeatWorker.java:73) ~[pixels-daemon-0.2.0-SNAPSHOT-full.jar:?]
	at io.pixelsdb.pixels.daemon.DaemonMain.main(DaemonMain.java:170) ~[pixels-daemon-0.2.0-SNAPSHOT-full.jar:?]
	at io.pixelsdb.pixels.daemon.PixelsRetina.main(PixelsRetina.java:30) ~[pixels-daemon-0.2.0-SNAPSHOT-full.jar:?]
Caused by: io.grpc.StatusRuntimeException: INTERNAL: Panic! This is a bug!
	at io.grpc.Status.asRuntimeException(Status.java:535) ~[ubuntu/:?]
	at io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onClose(ClientCalls.java:487) ~[ubuntu/:?]
	at io.grpc.internal.DelayedClientCall$DelayedListener$3.run(DelayedClientCall.java:470) ~[ubuntu/:?]
	at io.grpc.internal.DelayedClientCall$DelayedListener.delayOrExecute(DelayedClientCall.java:434) ~[ubuntu/:?]
	at io.grpc.internal.DelayedClientCall$DelayedListener.onClose(DelayedClientCall.java:467) ~[ubuntu/:?]
	at io.grpc.internal.ClientCallImpl.closeObserver(ClientCallImpl.java:563) ~[ubuntu/:?]
	at io.grpc.internal.ClientCallImpl.access$300(ClientCallImpl.java:70) ~[ubuntu/:?]
	at io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1StreamClosed.runInternal(ClientCallImpl.java:744) ~[ubuntu/:?]
	at io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1StreamClosed.runInContext(ClientCallImpl.java:723) ~[ubuntu/:?]
	at io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) ~[ubuntu/:?]
	at io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:133) ~[ubuntu/:?]
	at io.vertx.grpc.VertxChannelBuilder.lambda$build$1(VertxChannelBuilder.java:307) ~[pixels-daemon-0.2.0-SNAPSHOT-full.jar:?]
	at io.grpc.internal.SerializingExecutor.schedule(SerializingExecutor.java:102) ~[ubuntu/:?]
	at io.grpc.internal.SerializingExecutor.execute(SerializingExecutor.java:95) ~[ubuntu/:?]
	at io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl.closedInternal(ClientCallImpl.java:752) ~[ubuntu/:?]
	at io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl.closed(ClientCallImpl.java:688) ~[ubuntu/:?]
	at io.grpc.internal.RetriableStream$Sublistener$5.run(RetriableStream.java:986) ~[ubuntu/:?]
	at io.grpc.SynchronizationContext.drain(SynchronizationContext.java:95) ~[ubuntu/:?]
	at io.grpc.SynchronizationContext.execute(SynchronizationContext.java:127) ~[ubuntu/:?]
	at io.grpc.internal.RetriableStream$Sublistener.closed(RetriableStream.java:981) ~[ubuntu/:?]
	at io.grpc.internal.DelayedStream$DelayedStreamListener$4.run(DelayedStream.java:510) ~[ubuntu/:?]
	at io.grpc.internal.DelayedStream$DelayedStreamListener.drainPendingCallbacks(DelayedStream.java:535) ~[ubuntu/:?]
	at io.grpc.internal.DelayedStream.drainPendingCalls(DelayedStream.java:187) ~[ubuntu/:?]
	at io.grpc.internal.DelayedStream.access$100(DelayedStream.java:44) ~[ubuntu/:?]
	at io.grpc.internal.DelayedStream$4.run(DelayedStream.java:148) ~[ubuntu/:?]
	at io.vertx.grpc.VertxChannelBuilder.lambda$build$1(VertxChannelBuilder.java:307) ~[pixels-daemon-0.2.0-SNAPSHOT-full.jar:?]
	at io.grpc.internal.DelayedClientTransport.reprocess(DelayedClientTransport.java:311) ~[ubuntu/:?]
	at io.grpc.internal.ManagedChannelImpl.updateSubchannelPicker(ManagedChannelImpl.java:896) ~[ubuntu/:?]
	at io.grpc.internal.ManagedChannelImpl.panic(ManagedChannelImpl.java:882) ~[ubuntu/:?]
	at io.grpc.internal.ManagedChannelImpl$2.uncaughtException(ManagedChannelImpl.java:186) ~[ubuntu/:?]
	at io.grpc.SynchronizationContext.drain(SynchronizationContext.java:97) ~[ubuntu/:?]
	at io.grpc.SynchronizationContext.execute(SynchronizationContext.java:127) ~[ubuntu/:?]
	at io.grpc.internal.ManagedChannelImpl$NameResolverListener.onResult(ManagedChannelImpl.java:1870) ~[ubuntu/:?]
	at io.grpc.NameResolver$Listener2.onAddresses(NameResolver.java:215) ~[ubuntu/:?]
	at io.etcd.jetcd.resolver.AbstractNameResolver.doResolve(AbstractNameResolver.java:124) ~[pixels-daemon-0.2.0-SNAPSHOT-full.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) ~[?:?]
	at java.lang.Thread.run(Thread.java:1575) ~[?:?]
Caused by: java.lang.NoSuchMethodError: 'boolean io.grpc.internal.GrpcUtil.getFlag(java.lang.String, boolean)'
	at io.grpc.netty.NettyClientHandler.<clinit>(NettyClientHandler.java:101) ~[pixels-daemon-0.2.0-SNAPSHOT-full.jar:?]
	at io.grpc.netty.NettyClientTransport.start(NettyClientTransport.java:238) ~[pixels-daemon-0.2.0-SNAPSHOT-full.jar:?]
	at io.grpc.internal.ForwardingConnectionClientTransport.start(ForwardingConnectionClientTransport.java:34) ~[ubuntu/:?]
	at io.grpc.internal.ForwardingConnectionClientTransport.start(ForwardingConnectionClientTransport.java:34) ~[ubuntu/:?]
	at io.grpc.internal.InternalSubchannel.startNewTransport(InternalSubchannel.java:259) ~[ubuntu/:?]
	at io.grpc.internal.InternalSubchannel.access$400(InternalSubchannel.java:66) ~[ubuntu/:?]
	at io.grpc.internal.InternalSubchannel$2.run(InternalSubchannel.java:201) ~[ubuntu/:?]
	at io.grpc.SynchronizationContext.drain(SynchronizationContext.java:95) ~[ubuntu/:?]
	at io.grpc.SynchronizationContext.execute(SynchronizationContext.java:127) ~[ubuntu/:?]
	at io.grpc.internal.ManagedChannelImpl$NameResolverListener.onResult(ManagedChannelImpl.java:1870) ~[ubuntu/:?]
	at io.grpc.NameResolver$Listener2.onAddresses(NameResolver.java:215) ~[ubuntu/:?]
	at io.etcd.jetcd.resolver.AbstractNameResolver.doResolve(AbstractNameResolver.java:124) ~[pixels-daemon-0.2.0-SNAPSHOT-full.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) ~[?:?]
	at java.lang.Thread.run(Thread.java:1575) ~[?:?]
2026-01-06 05:47:47,134 [io.pixelsdb.pixels.daemon.heartbeat.HeartbeatWorker]-[ERROR] No heartbeat coordinator found, exit...
2026-01-06 05:47:47,135 [io.pixelsdb.pixels.daemon.heartbeat.HeartbeatWorker]-[INFO] Starting heartbeat worker
2026-01-06 05:47:47,135 [io.pixelsdb.pixels.daemon.heartbeat.HeartbeatWorker]-[ERROR] Heartbeat worker initialization failed, exit now...
2026-01-06 05:47:47,439 [io.pixelsdb.pixels.daemon.retina.RetinaServer]-[INFO] Metadata server is ready
2026-01-06 05:47:47,461 [io.pixelsdb.pixels.daemon.retina.RetinaServerImpl]-[INFO] Pre-loading checkpoints...
2026-01-06 05:47:47,467 [io.pixelsdb.pixels.retina.RetinaResourceManager]-[INFO] Loading system state from GC checkpoint: 5007
2026-01-06 05:49:09,099 [io.pixelsdb.pixels.common.utils.ShutdownHookManager]-[INFO] Invoke parallel shutdown hook from io.pixelsdb.pixels.daemon.DaemonMain
2026-01-06 05:49:09,099 [io.pixelsdb.pixels.daemon.heartbeat.HeartbeatWorker]-[INFO] Shutting down heartbeat worker...
2026-01-06 05:49:09,100 [io.pixelsdb.pixels.common.utils.ShutdownHookManager]-[INFO] Invoke serial shutdown hook from io.pixelsdb.pixels.common.physical.StorageFactory
2026-01-06 05:49:10,129 [io.pixelsdb.pixels.common.utils.ShutdownHookManager]-[INFO] Invoke parallel shutdown hook from io.pixelsdb.pixels.daemon.DaemonMain
2026-01-06 05:49:10,129 [io.pixelsdb.pixels.daemon.heartbeat.HeartbeatWorker]-[INFO] Shutting down heartbeat worker...
2026-01-06 05:49:10,859 [io.pixelsdb.pixels.common.utils.ShutdownHookManager]-[INFO] Invoke parallel shutdown hook from io.pixelsdb.pixels.daemon.DaemonMain
2026-01-06 05:51:30,078 [io.pixelsdb.pixels.daemon.Daemon]-[INFO] starting daemon thread...
2026-01-06 05:51:31,085 [io.pixelsdb.pixels.common.utils.EtcdUtil]-[INFO] Using etcd host: localhost
2026-01-06 05:51:31,610 [io.pixelsdb.pixels.daemon.heartbeat.HeartbeatCoordinator]-[INFO] Heartbeat coordinator on ip-172-31-1-226 is initialized
2026-01-06 05:51:31,611 [io.pixelsdb.pixels.daemon.heartbeat.HeartbeatCoordinator]-[INFO] Starting heartbeat coordinator
2026-01-06 05:51:31,611 [io.pixelsdb.pixels.daemon.heartbeat.HeartbeatCoordinator]-[INFO] Heartbeat coordinator is running
2026-01-06 05:51:32,250 [io.pixelsdb.pixels.daemon.node.NodeServiceImpl]-[INFO] Initial hash ring loaded with 0 physical nodes and 128 virtual nodes.
2026-01-06 05:51:35,701 [io.pixelsdb.pixels.daemon.Daemon]-[INFO] starting daemon thread...
2026-01-06 05:51:36,707 [io.pixelsdb.pixels.common.utils.EtcdUtil]-[INFO] Using etcd host: localhost
2026-01-06 05:51:37,100 [io.pixelsdb.pixels.common.utils.EtcdUtil]-[ERROR] error when get key-values by prefix.
java.util.concurrent.ExecutionException: io.grpc.StatusRuntimeException: INTERNAL: Panic! This is a bug!
	at java.util.concurrent.CompletableFuture.wrapInExecutionException(CompletableFuture.java:345) ~[?:?]
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:440) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2117) ~[?:?]
	at io.pixelsdb.pixels.common.utils.EtcdUtil.getKeyValueByPrefix(EtcdUtil.java:160) ~[pixels-daemon-0.2.0-SNAPSHOT-full.jar:?]
	at io.pixelsdb.pixels.daemon.heartbeat.HeartbeatWorker.initialize(HeartbeatWorker.java:88) ~[pixels-daemon-0.2.0-SNAPSHOT-full.jar:?]
	at io.pixelsdb.pixels.daemon.heartbeat.HeartbeatWorker.<init>(HeartbeatWorker.java:73) ~[pixels-daemon-0.2.0-SNAPSHOT-full.jar:?]
	at io.pixelsdb.pixels.daemon.DaemonMain.main(DaemonMain.java:146) ~[pixels-daemon-0.2.0-SNAPSHOT-full.jar:?]
	at io.pixelsdb.pixels.daemon.PixelsWorker.main(PixelsWorker.java:30) ~[pixels-daemon-0.2.0-SNAPSHOT-full.jar:?]
Caused by: io.grpc.StatusRuntimeException: INTERNAL: Panic! This is a bug!
	at io.grpc.Status.asRuntimeException(Status.java:535) ~[ubuntu/:?]
	at io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onClose(ClientCalls.java:487) ~[ubuntu/:?]
	at io.grpc.internal.DelayedClientCall$DelayedListener$3.run(DelayedClientCall.java:470) ~[ubuntu/:?]
	at io.grpc.internal.DelayedClientCall$DelayedListener.delayOrExecute(DelayedClientCall.java:434) ~[ubuntu/:?]
	at io.grpc.internal.DelayedClientCall$DelayedListener.onClose(DelayedClientCall.java:467) ~[ubuntu/:?]
	at io.grpc.internal.ClientCallImpl.closeObserver(ClientCallImpl.java:563) ~[ubuntu/:?]
	at io.grpc.internal.ClientCallImpl.access$300(ClientCallImpl.java:70) ~[ubuntu/:?]
	at io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1StreamClosed.runInternal(ClientCallImpl.java:744) ~[ubuntu/:?]
	at io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1StreamClosed.runInContext(ClientCallImpl.java:723) ~[ubuntu/:?]
	at io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) ~[ubuntu/:?]
	at io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:133) ~[ubuntu/:?]
	at io.vertx.grpc.VertxChannelBuilder.lambda$build$1(VertxChannelBuilder.java:307) ~[pixels-daemon-0.2.0-SNAPSHOT-full.jar:?]
	at io.grpc.internal.SerializingExecutor.schedule(SerializingExecutor.java:102) ~[ubuntu/:?]
	at io.grpc.internal.SerializingExecutor.execute(SerializingExecutor.java:95) ~[ubuntu/:?]
	at io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl.closedInternal(ClientCallImpl.java:752) ~[ubuntu/:?]
	at io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl.closed(ClientCallImpl.java:688) ~[ubuntu/:?]
	at io.grpc.internal.RetriableStream$Sublistener$5.run(RetriableStream.java:986) ~[ubuntu/:?]
	at io.grpc.SynchronizationContext.drain(SynchronizationContext.java:95) ~[ubuntu/:?]
	at io.grpc.SynchronizationContext.execute(SynchronizationContext.java:127) ~[ubuntu/:?]
	at io.grpc.internal.RetriableStream$Sublistener.closed(RetriableStream.java:981) ~[ubuntu/:?]
	at io.grpc.internal.DelayedStream$DelayedStreamListener$4.run(DelayedStream.java:510) ~[ubuntu/:?]
	at io.grpc.internal.DelayedStream$DelayedStreamListener.drainPendingCallbacks(DelayedStream.java:535) ~[ubuntu/:?]
	at io.grpc.internal.DelayedStream.drainPendingCalls(DelayedStream.java:187) ~[ubuntu/:?]
	at io.grpc.internal.DelayedStream.access$100(DelayedStream.java:44) ~[ubuntu/:?]
	at io.grpc.internal.DelayedStream$4.run(DelayedStream.java:148) ~[ubuntu/:?]
	at io.vertx.grpc.VertxChannelBuilder.lambda$build$1(VertxChannelBuilder.java:307) ~[pixels-daemon-0.2.0-SNAPSHOT-full.jar:?]
	at io.grpc.internal.DelayedClientTransport.reprocess(DelayedClientTransport.java:311) ~[ubuntu/:?]
	at io.grpc.internal.ManagedChannelImpl.updateSubchannelPicker(ManagedChannelImpl.java:896) ~[ubuntu/:?]
	at io.grpc.internal.ManagedChannelImpl.panic(ManagedChannelImpl.java:882) ~[ubuntu/:?]
	at io.grpc.internal.ManagedChannelImpl$2.uncaughtException(ManagedChannelImpl.java:186) ~[ubuntu/:?]
	at io.grpc.SynchronizationContext.drain(SynchronizationContext.java:97) ~[ubuntu/:?]
	at io.grpc.SynchronizationContext.execute(SynchronizationContext.java:127) ~[ubuntu/:?]
	at io.grpc.internal.ManagedChannelImpl$NameResolverListener.onResult(ManagedChannelImpl.java:1870) ~[ubuntu/:?]
	at io.grpc.NameResolver$Listener2.onAddresses(NameResolver.java:215) ~[ubuntu/:?]
	at io.etcd.jetcd.resolver.AbstractNameResolver.doResolve(AbstractNameResolver.java:124) ~[pixels-daemon-0.2.0-SNAPSHOT-full.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) ~[?:?]
	at java.lang.Thread.run(Thread.java:1575) ~[?:?]
Caused by: java.lang.NoSuchMethodError: 'boolean io.grpc.internal.GrpcUtil.getFlag(java.lang.String, boolean)'
	at io.grpc.netty.NettyClientHandler.<clinit>(NettyClientHandler.java:101) ~[pixels-daemon-0.2.0-SNAPSHOT-full.jar:?]
	at io.grpc.netty.NettyClientTransport.start(NettyClientTransport.java:238) ~[pixels-daemon-0.2.0-SNAPSHOT-full.jar:?]
	at io.grpc.internal.ForwardingConnectionClientTransport.start(ForwardingConnectionClientTransport.java:34) ~[ubuntu/:?]
	at io.grpc.internal.ForwardingConnectionClientTransport.start(ForwardingConnectionClientTransport.java:34) ~[ubuntu/:?]
	at io.grpc.internal.InternalSubchannel.startNewTransport(InternalSubchannel.java:259) ~[ubuntu/:?]
	at io.grpc.internal.InternalSubchannel.access$400(InternalSubchannel.java:66) ~[ubuntu/:?]
	at io.grpc.internal.InternalSubchannel$2.run(InternalSubchannel.java:201) ~[ubuntu/:?]
	at io.grpc.SynchronizationContext.drain(SynchronizationContext.java:95) ~[ubuntu/:?]
	at io.grpc.SynchronizationContext.execute(SynchronizationContext.java:127) ~[ubuntu/:?]
	at io.grpc.internal.ManagedChannelImpl$NameResolverListener.onResult(ManagedChannelImpl.java:1870) ~[ubuntu/:?]
	at io.grpc.NameResolver$Listener2.onAddresses(NameResolver.java:215) ~[ubuntu/:?]
	at io.etcd.jetcd.resolver.AbstractNameResolver.doResolve(AbstractNameResolver.java:124) ~[pixels-daemon-0.2.0-SNAPSHOT-full.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) ~[?:?]
	at java.lang.Thread.run(Thread.java:1575) ~[?:?]
2026-01-06 05:51:37,109 [io.pixelsdb.pixels.daemon.heartbeat.HeartbeatWorker]-[ERROR] No heartbeat coordinator found, exit...
2026-01-06 05:51:37,110 [io.pixelsdb.pixels.daemon.heartbeat.HeartbeatWorker]-[INFO] Starting heartbeat worker
2026-01-06 05:51:37,110 [io.pixelsdb.pixels.daemon.heartbeat.HeartbeatWorker]-[ERROR] Heartbeat worker initialization failed, exit now...
2026-01-06 05:51:41,018 [io.pixelsdb.pixels.daemon.Daemon]-[INFO] starting daemon thread...
2026-01-06 05:51:42,025 [io.pixelsdb.pixels.common.utils.EtcdUtil]-[INFO] Using etcd host: localhost
2026-01-06 05:51:42,415 [io.pixelsdb.pixels.common.utils.EtcdUtil]-[ERROR] error when get key-values by prefix.
java.util.concurrent.ExecutionException: io.grpc.StatusRuntimeException: INTERNAL: Panic! This is a bug!
	at java.util.concurrent.CompletableFuture.wrapInExecutionException(CompletableFuture.java:345) ~[?:?]
	at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:440) ~[?:?]
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2117) ~[?:?]
	at io.pixelsdb.pixels.common.utils.EtcdUtil.getKeyValueByPrefix(EtcdUtil.java:160) ~[pixels-daemon-0.2.0-SNAPSHOT-full.jar:?]
	at io.pixelsdb.pixels.daemon.heartbeat.HeartbeatWorker.initialize(HeartbeatWorker.java:88) ~[pixels-daemon-0.2.0-SNAPSHOT-full.jar:?]
	at io.pixelsdb.pixels.daemon.heartbeat.HeartbeatWorker.<init>(HeartbeatWorker.java:73) ~[pixels-daemon-0.2.0-SNAPSHOT-full.jar:?]
	at io.pixelsdb.pixels.daemon.DaemonMain.main(DaemonMain.java:170) ~[pixels-daemon-0.2.0-SNAPSHOT-full.jar:?]
	at io.pixelsdb.pixels.daemon.PixelsRetina.main(PixelsRetina.java:30) ~[pixels-daemon-0.2.0-SNAPSHOT-full.jar:?]
Caused by: io.grpc.StatusRuntimeException: INTERNAL: Panic! This is a bug!
	at io.grpc.Status.asRuntimeException(Status.java:535) ~[ubuntu/:?]
	at io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onClose(ClientCalls.java:487) ~[ubuntu/:?]
	at io.grpc.internal.DelayedClientCall$DelayedListener$3.run(DelayedClientCall.java:470) ~[ubuntu/:?]
	at io.grpc.internal.DelayedClientCall$DelayedListener.delayOrExecute(DelayedClientCall.java:434) ~[ubuntu/:?]
	at io.grpc.internal.DelayedClientCall$DelayedListener.onClose(DelayedClientCall.java:467) ~[ubuntu/:?]
	at io.grpc.internal.ClientCallImpl.closeObserver(ClientCallImpl.java:563) ~[ubuntu/:?]
	at io.grpc.internal.ClientCallImpl.access$300(ClientCallImpl.java:70) ~[ubuntu/:?]
	at io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1StreamClosed.runInternal(ClientCallImpl.java:744) ~[ubuntu/:?]
	at io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1StreamClosed.runInContext(ClientCallImpl.java:723) ~[ubuntu/:?]
	at io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) ~[ubuntu/:?]
	at io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:133) ~[ubuntu/:?]
	at io.vertx.grpc.VertxChannelBuilder.lambda$build$1(VertxChannelBuilder.java:307) ~[pixels-daemon-0.2.0-SNAPSHOT-full.jar:?]
	at io.grpc.internal.SerializingExecutor.schedule(SerializingExecutor.java:102) ~[ubuntu/:?]
	at io.grpc.internal.SerializingExecutor.execute(SerializingExecutor.java:95) ~[ubuntu/:?]
	at io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl.closedInternal(ClientCallImpl.java:752) ~[ubuntu/:?]
	at io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl.closed(ClientCallImpl.java:688) ~[ubuntu/:?]
	at io.grpc.internal.RetriableStream$Sublistener$5.run(RetriableStream.java:986) ~[ubuntu/:?]
	at io.grpc.SynchronizationContext.drain(SynchronizationContext.java:95) ~[ubuntu/:?]
	at io.grpc.SynchronizationContext.execute(SynchronizationContext.java:127) ~[ubuntu/:?]
	at io.grpc.internal.RetriableStream$Sublistener.closed(RetriableStream.java:981) ~[ubuntu/:?]
	at io.grpc.internal.DelayedStream$DelayedStreamListener$4.run(DelayedStream.java:510) ~[ubuntu/:?]
	at io.grpc.internal.DelayedStream$DelayedStreamListener.drainPendingCallbacks(DelayedStream.java:535) ~[ubuntu/:?]
	at io.grpc.internal.DelayedStream.drainPendingCalls(DelayedStream.java:187) ~[ubuntu/:?]
	at io.grpc.internal.DelayedStream.access$100(DelayedStream.java:44) ~[ubuntu/:?]
	at io.grpc.internal.DelayedStream$4.run(DelayedStream.java:148) ~[ubuntu/:?]
	at io.vertx.grpc.VertxChannelBuilder.lambda$build$1(VertxChannelBuilder.java:307) ~[pixels-daemon-0.2.0-SNAPSHOT-full.jar:?]
	at io.grpc.internal.DelayedClientTransport.reprocess(DelayedClientTransport.java:311) ~[ubuntu/:?]
	at io.grpc.internal.ManagedChannelImpl.updateSubchannelPicker(ManagedChannelImpl.java:896) ~[ubuntu/:?]
	at io.grpc.internal.ManagedChannelImpl.panic(ManagedChannelImpl.java:882) ~[ubuntu/:?]
	at io.grpc.internal.ManagedChannelImpl$2.uncaughtException(ManagedChannelImpl.java:186) ~[ubuntu/:?]
	at io.grpc.SynchronizationContext.drain(SynchronizationContext.java:97) ~[ubuntu/:?]
	at io.grpc.SynchronizationContext.execute(SynchronizationContext.java:127) ~[ubuntu/:?]
	at io.grpc.internal.ManagedChannelImpl$NameResolverListener.onResult(ManagedChannelImpl.java:1870) ~[ubuntu/:?]
	at io.grpc.NameResolver$Listener2.onAddresses(NameResolver.java:215) ~[ubuntu/:?]
	at io.etcd.jetcd.resolver.AbstractNameResolver.doResolve(AbstractNameResolver.java:124) ~[pixels-daemon-0.2.0-SNAPSHOT-full.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) ~[?:?]
	at java.lang.Thread.run(Thread.java:1575) ~[?:?]
Caused by: java.lang.NoSuchMethodError: 'boolean io.grpc.internal.GrpcUtil.getFlag(java.lang.String, boolean)'
	at io.grpc.netty.NettyClientHandler.<clinit>(NettyClientHandler.java:101) ~[pixels-daemon-0.2.0-SNAPSHOT-full.jar:?]
	at io.grpc.netty.NettyClientTransport.start(NettyClientTransport.java:238) ~[pixels-daemon-0.2.0-SNAPSHOT-full.jar:?]
	at io.grpc.internal.ForwardingConnectionClientTransport.start(ForwardingConnectionClientTransport.java:34) ~[ubuntu/:?]
	at io.grpc.internal.ForwardingConnectionClientTransport.start(ForwardingConnectionClientTransport.java:34) ~[ubuntu/:?]
	at io.grpc.internal.InternalSubchannel.startNewTransport(InternalSubchannel.java:259) ~[ubuntu/:?]
	at io.grpc.internal.InternalSubchannel.access$400(InternalSubchannel.java:66) ~[ubuntu/:?]
	at io.grpc.internal.InternalSubchannel$2.run(InternalSubchannel.java:201) ~[ubuntu/:?]
	at io.grpc.SynchronizationContext.drain(SynchronizationContext.java:95) ~[ubuntu/:?]
	at io.grpc.SynchronizationContext.execute(SynchronizationContext.java:127) ~[ubuntu/:?]
	at io.grpc.internal.ManagedChannelImpl$NameResolverListener.onResult(ManagedChannelImpl.java:1870) ~[ubuntu/:?]
	at io.grpc.NameResolver$Listener2.onAddresses(NameResolver.java:215) ~[ubuntu/:?]
	at io.etcd.jetcd.resolver.AbstractNameResolver.doResolve(AbstractNameResolver.java:124) ~[pixels-daemon-0.2.0-SNAPSHOT-full.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) ~[?:?]
	at java.lang.Thread.run(Thread.java:1575) ~[?:?]
2026-01-06 05:51:42,424 [io.pixelsdb.pixels.daemon.heartbeat.HeartbeatWorker]-[ERROR] No heartbeat coordinator found, exit...
2026-01-06 05:51:42,425 [io.pixelsdb.pixels.daemon.heartbeat.HeartbeatWorker]-[INFO] Starting heartbeat worker
2026-01-06 05:51:42,425 [io.pixelsdb.pixels.daemon.heartbeat.HeartbeatWorker]-[ERROR] Heartbeat worker initialization failed, exit now...
2026-01-06 05:51:42,746 [io.pixelsdb.pixels.daemon.retina.RetinaServer]-[INFO] Metadata server is ready
2026-01-06 05:51:42,770 [io.pixelsdb.pixels.daemon.retina.RetinaServerImpl]-[INFO] Pre-loading checkpoints...
2026-01-06 05:51:42,776 [io.pixelsdb.pixels.retina.RetinaResourceManager]-[INFO] Loading system state from GC checkpoint: 5007

Metadata

Metadata

Assignees

No one assigned

    Labels

    No labels
    No labels

    Type

    No type

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions