feat: create fullNode method

.gitignore

@@ -6,5 +6,8 @@ node_modules/
 cache/
 artifcats/
 
+# Zkopru database
+*.sqlite
+
 .build-cache
 .vscode/*

dockerfiles/hardhat.config.ts

@@ -14,13 +14,14 @@ if (url !== '') {
         blockNumber: blockNumber == 0 ? undefined : blockNumber
       },
       chainId,
+      loggingEnabled: true,
     }
   }
 }
 
 const config: HardhatUserConfig = {
   solidity: "0.7.4",
-  networks: { ...network }
+  networks: { ...network },
 }
 
 module.exports = config

package.json

@@ -14,8 +14,11 @@
   },
   "dependencies": {
     "@types/bn.js": "^5.1.0",
+    "@types/cli-progress": "^3.9.2",
     "bn.js": "5.2",
+    "cli-progress": "^3.10.0",
     "dockerode": "^3.3.1",
+    "ethers": "^5.6.2",
     "hardhat": "^2.8.3",
     "soltypes": "1.3.5"
   },
@@ -27,7 +30,7 @@
     "link-module-alias": "^1.2.0",
     "shx": "^0.3.4",
     "ts-jest": "^27.1.3",
-    "ts-node": "^10.5.0",
+    "ts-node": "^10.7.0",
     "typescript": "^4.5.5",
     "web3": "1.2.11"
   },

scripts/createDatabase.ts

@@ -0,0 +1,79 @@
+import Docker from 'dockerode'
+import { BigNumber } from 'ethers'
+import { SingleBar } from 'cli-progress'
+import { getContainers, removeContainer, createFullNode } from '../src/utils'
+import { sleep } from '~zkopru/utils'
+
+const docker = new Docker({ socketPath: '/var/run/docker.sock' })
+
+const containerName = process.env.CONTAINER_NAME ?? `zkopru-hardhat-debug`
+const sourceNode = process.env.URL
+const blockNumber = process.env.BLOCK_NUMBER
+const zkopruAddress = process.env.ZKOPRU_ADDRESS
+
+const containerConfig = {
+  Image: `zkopru-debug/hardhat:latest`,
+  name: containerName,
+  AttachStdin: true,
+  AttachStdout: true,
+  AttachStderr: true,
+  Tty: true,
+  Env: [`URL=${sourceNode}`],
+  HostConfig: {
+    PortBindings: {
+      "8545/tcp": [{ HostPort: "8545" }]
+    }
+  },
+  ExposedPorts: { "8545/tcp": {} }
+}
+
+async function main() {
+  // check configurations
+  if (!zkopruAddress) throw Error(`Zkopru address not set`)
+  if (blockNumber && blockNumber != 'latest') {
+    containerConfig.Env.push(`BLOCK_NUMBER=${blockNumber}`)
+  }
+
+  // create hardhat container
+  const currentContainerId = await getContainers(containerName)
+  if (currentContainerId) {
+    console.log(`Found running '${containerName}' container`) 
+    console.log(`Stop and remove container: ${currentContainerId.slice(0, 8)}`)
+    await removeContainer(currentContainerId)
+  }
+  const hardhatContainer = await docker.createContainer(containerConfig)
+  await hardhatContainer.start()
+
+  // wait container is running
+  let containerInfo = await hardhatContainer.inspect()
+  while (!containerInfo.State.Running) {
+    await sleep(500)
+    containerInfo = await hardhatContainer.inspect()
+  }
+
+  // start zkopru full node
+  const fullNode = await createFullNode(`http://localhost:8545`, zkopruAddress, blockNumber)
+  fullNode.start()
+
+  let proposedBlocks: BigNumber = await fullNode.synchronizer.l1Contract.zkopru.proposedBlocks()
+  await sleep(1000)
+
+  // logging syncing progress
+  let isSyncing = true
+  const bar = new SingleBar({ format :  `Syncing | [{bar}] | {percentage}% | {value}/{total} blocks`})
+  bar.start(proposedBlocks.toNumber(), 0)
+  while (isSyncing) {
+    proposedBlocks = await fullNode.synchronizer.l1Contract.zkopru.proposedBlocks()
+    bar.update(fullNode.synchronizer.latestProcessed ?? 0)
+    isSyncing = fullNode.synchronizer.isSynced() == false
+    await sleep(1000)
+  }
+
+  bar.stop()
+  await fullNode.stop()
+  await removeContainer(hardhatContainer.id)
+  console.log(`Sync and zkopru data download complete, close now`)
+  process.exit(1)
+}
+
+main()

src/hre-controller.ts

@@ -1,4 +1,3 @@
-// import hre from 'hardhat'
 import { HttpNetworkConfig, EthereumProvider } from 'hardhat/types'
 import { createProvider } from 'hardhat/internal/core/providers/construction'
 import { rpcBlock } from 'hardhat/internal/core/jsonrpc/types/output/block'

src/utils.ts

@@ -1,4 +1,9 @@
+import { FullNode } from '~zkopru/core'
+import { DB, schema } from '~zkopru/database'
+import { ConnectionInfo } from '@ethersproject/web'
+import { JsonRpcProvider } from '@ethersproject/providers'
 import Docker, { Container } from 'dockerode'
+import { SQLiteConnector } from '~zkopru/database/dist/node'
 
 const ImageName = process.env.DOCKER_IMAGE_NAME ?? 'zkopru-debug/hardhat'
 const ImageTag = process.env.DOCKER_IMAGE_TAG ?? 'latest'
@@ -63,3 +68,37 @@ export async function runForkedChain(url?: string, blockNumber?: number, chainId
 
   return hardhatContainer
 }
+
+export async function createFullNode(nodeUrl: string, zkopruAddress: string, blockNumber?: string | 'latest') {
+  const connectionInfo: ConnectionInfo = {
+    url: nodeUrl,
+    timeout: 300000
+  }
+  const provider = new JsonRpcProvider(connectionInfo)
+
+  async function waitConnection() {
+    return new Promise<void>(async res => {
+      if (await provider.ready) res()
+      provider.on('connect', res)
+    })
+  }
+
+  await waitConnection()
+
+  // configure database
+  let outputFile: string = ":memory:"
+  if (blockNumber) {
+    outputFile = `database-${blockNumber}.sqlite`
+  }
+  if (blockNumber == 'latest') {
+    const latestBlockNumber = await provider.getBlockNumber()
+    outputFile = `database-${latestBlockNumber}.sqlite`
+  }
+
+  const db: DB = await SQLiteConnector.create(schema, outputFile)
+  return FullNode.new({
+    address: zkopruAddress,
+    provider,
+    db
+  })
+}

src/zkopru-data.ts

@@ -0,0 +1,105 @@
+import { Block as BlockCore } from '../zkopru/packages/core'
+// import { Block as OriginBlock } from '../zkopru/packages/client/src/types'
+import { FullNode } from '../zkopru/packages/core'
+
+
+// Prefix e means Extend
+// TODO: consider using this TreeNode
+// interface TreeNode extends OriginTreeNode {
+//   utxoType: number // Deposit, Tx, Withdrawal
+// }
+
+// interface Block extends OriginBlock {
+//   slash?: Slash
+// }
+
+interface L2blockHashes {
+  [CanonicalBlockNumber: number]: string[] // Block Hash
+}
+
+class ZkopruData {
+  fullNode: FullNode
+
+  L2blockHashes: L2blockHashes // Fast
+
+  lastUpdatedBlockCount: number
+
+  constructor(fullNode: FullNode) {
+      this.fullNode = fullNode
+      this.L2blockHashes = {}
+      this.lastUpdatedBlockCount = -1
+  }
+
+  // Update state this data class every event recieved on full node
+  async updateL2BlockHashes() {
+      const { db } = this.fullNode
+      const totalProposalCount = await db.count('Proposal', {
+          include: { block: true }
+      })
+
+      // update 'L2blocks' double size of updateWindow
+      const updateBlockNum = totalProposalCount - this.lastUpdatedBlockCount
+      const updateWindow = Math.max(this.lastUpdatedBlockCount - updateBlockNum, 0)
+
+      if (updateWindow > 0) {
+          // const targetProposals: {proposalNum: number}[] = []
+          // for (const num of [...Array(updateWindow).keys()]) {
+          //     targetProposals.push({proposalNum: startBlockNum + num})   
+          // }
+          const updatingProposals = await db.findMany('Proposal', {
+              where: {},
+              orderBy: { proposalNum: 'desc' },
+              limit: updateWindow
+          })
+
+          // update L2blocks data
+          for (const proposal of updatingProposals) {
+              const l2block = this.L2blockHashes[proposal.canonicalNum]
+              if (!l2block) {
+                  this.L2blockHashes[proposal.canonicalNum] = [proposal.hash]
+                  this.lastUpdatedBlockCount++
+                  continue
+              }
+              if (!l2block.includes(proposal.hash)) {
+                  l2block.push(proposal.hash)
+                  this.lastUpdatedBlockCount++
+              }
+          }
+      }
+  }
+
+  private getBlockHashesByNumber(CanonicalBlockNumber: number) {
+      if (this.L2blockHashes[CanonicalBlockNumber]) return undefined
+      return this.L2blockHashes[CanonicalBlockNumber]
+  }
+
+  // TODO: full data export
+  // type from api-client 
+  async getBlockData(CanonicalBlockNumber: number) {
+      const { db } = this.fullNode
+      const targetHashes = this.getBlockHashesByNumber(CanonicalBlockNumber)
+
+      let blocks: any[] = []
+      if (targetHashes && targetHashes.length > 0) {
+          for (const blockHash of targetHashes) {
+              // get data from database
+              // TODO: get Include related
+              const [ proposal, header, slash ] = await Promise.all([
+                  db.findOne('Proposal', { where: { hash: blockHash } }),
+                  db.findOne('Header', { where: { hash: blockHash } }),
+                  db.findOne('Slash',  { where: { hash: blockHash } })
+              ])
+            //   db.findOne(`Block`, { where: { hash: blockhash }, include: { block: true, })
+
+              // body parsing 
+              const { proposalData, ...remains } = proposal
+              const blockData = BlockCore.from(proposalData)
+              blocks.push({ ...remains, header, blockData, slash})
+          }
+      }
+
+      return JSON.stringify(blocks)
+  }
+}
+
+export default ZkopruData