diff --git a/.gitignore b/.gitignore index 12aaa39..e5fb59b 100644 --- a/.gitignore +++ b/.gitignore @@ -6,5 +6,8 @@ node_modules/ cache/ artifcats/ +# Zkopru database +*.sqlite + .build-cache .vscode/* diff --git a/dockerfiles/hardhat.config.ts b/dockerfiles/hardhat.config.ts index 92507f2..8321b8c 100644 --- a/dockerfiles/hardhat.config.ts +++ b/dockerfiles/hardhat.config.ts @@ -14,13 +14,14 @@ if (url !== '') { blockNumber: blockNumber == 0 ? undefined : blockNumber }, chainId, + loggingEnabled: true, } } } const config: HardhatUserConfig = { solidity: "0.7.4", - networks: { ...network } + networks: { ...network }, } module.exports = config diff --git a/package.json b/package.json index 01bff89..14c7fb9 100644 --- a/package.json +++ b/package.json @@ -14,8 +14,11 @@ }, "dependencies": { "@types/bn.js": "^5.1.0", + "@types/cli-progress": "^3.9.2", "bn.js": "5.2", + "cli-progress": "^3.10.0", "dockerode": "^3.3.1", + "ethers": "^5.6.2", "hardhat": "^2.8.3", "soltypes": "1.3.5" }, @@ -27,7 +30,7 @@ "link-module-alias": "^1.2.0", "shx": "^0.3.4", "ts-jest": "^27.1.3", - "ts-node": "^10.5.0", + "ts-node": "^10.7.0", "typescript": "^4.5.5", "web3": "1.2.11" }, diff --git a/scripts/createDatabase.ts b/scripts/createDatabase.ts new file mode 100644 index 0000000..674a63c --- /dev/null +++ b/scripts/createDatabase.ts @@ -0,0 +1,79 @@ +import Docker from 'dockerode' +import { BigNumber } from 'ethers' +import { SingleBar } from 'cli-progress' +import { getContainers, removeContainer, createFullNode } from '../src/utils' +import { sleep } from '~zkopru/utils' + +const docker = new Docker({ socketPath: '/var/run/docker.sock' }) + +const containerName = process.env.CONTAINER_NAME ?? `zkopru-hardhat-debug` +const sourceNode = process.env.URL +const blockNumber = process.env.BLOCK_NUMBER +const zkopruAddress = process.env.ZKOPRU_ADDRESS + +const containerConfig = { + Image: `zkopru-debug/hardhat:latest`, + name: containerName, + AttachStdin: true, + AttachStdout: true, + AttachStderr: true, + Tty: true, + Env: [`URL=${sourceNode}`], + HostConfig: { + PortBindings: { + "8545/tcp": [{ HostPort: "8545" }] + } + }, + ExposedPorts: { "8545/tcp": {} } +} + +async function main() { + // check configurations + if (!zkopruAddress) throw Error(`Zkopru address not set`) + if (blockNumber && blockNumber != 'latest') { + containerConfig.Env.push(`BLOCK_NUMBER=${blockNumber}`) + } + + // create hardhat container + const currentContainerId = await getContainers(containerName) + if (currentContainerId) { + console.log(`Found running '${containerName}' container`) + console.log(`Stop and remove container: ${currentContainerId.slice(0, 8)}`) + await removeContainer(currentContainerId) + } + const hardhatContainer = await docker.createContainer(containerConfig) + await hardhatContainer.start() + + // wait container is running + let containerInfo = await hardhatContainer.inspect() + while (!containerInfo.State.Running) { + await sleep(500) + containerInfo = await hardhatContainer.inspect() + } + + // start zkopru full node + const fullNode = await createFullNode(`http://localhost:8545`, zkopruAddress, blockNumber) + fullNode.start() + + let proposedBlocks: BigNumber = await fullNode.synchronizer.l1Contract.zkopru.proposedBlocks() + await sleep(1000) + + // logging syncing progress + let isSyncing = true + const bar = new SingleBar({ format : `Syncing | [{bar}] | {percentage}% | {value}/{total} blocks`}) + bar.start(proposedBlocks.toNumber(), 0) + while (isSyncing) { + proposedBlocks = await fullNode.synchronizer.l1Contract.zkopru.proposedBlocks() + bar.update(fullNode.synchronizer.latestProcessed ?? 0) + isSyncing = fullNode.synchronizer.isSynced() == false + await sleep(1000) + } + + bar.stop() + await fullNode.stop() + await removeContainer(hardhatContainer.id) + console.log(`Sync and zkopru data download complete, close now`) + process.exit(1) +} + +main() diff --git a/src/hre-controller.ts b/src/hre-controller.ts index f8d9cd0..46bd468 100644 --- a/src/hre-controller.ts +++ b/src/hre-controller.ts @@ -1,4 +1,3 @@ -// import hre from 'hardhat' import { HttpNetworkConfig, EthereumProvider } from 'hardhat/types' import { createProvider } from 'hardhat/internal/core/providers/construction' import { rpcBlock } from 'hardhat/internal/core/jsonrpc/types/output/block' diff --git a/src/utils.ts b/src/utils.ts index 0da7af6..9b9032a 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,4 +1,9 @@ +import { FullNode } from '~zkopru/core' +import { DB, schema } from '~zkopru/database' +import { ConnectionInfo } from '@ethersproject/web' +import { JsonRpcProvider } from '@ethersproject/providers' import Docker, { Container } from 'dockerode' +import { SQLiteConnector } from '~zkopru/database/dist/node' const ImageName = process.env.DOCKER_IMAGE_NAME ?? 'zkopru-debug/hardhat' const ImageTag = process.env.DOCKER_IMAGE_TAG ?? 'latest' @@ -63,3 +68,37 @@ export async function runForkedChain(url?: string, blockNumber?: number, chainId return hardhatContainer } + +export async function createFullNode(nodeUrl: string, zkopruAddress: string, blockNumber?: string | 'latest') { + const connectionInfo: ConnectionInfo = { + url: nodeUrl, + timeout: 300000 + } + const provider = new JsonRpcProvider(connectionInfo) + + async function waitConnection() { + return new Promise(async res => { + if (await provider.ready) res() + provider.on('connect', res) + }) + } + + await waitConnection() + + // configure database + let outputFile: string = ":memory:" + if (blockNumber) { + outputFile = `database-${blockNumber}.sqlite` + } + if (blockNumber == 'latest') { + const latestBlockNumber = await provider.getBlockNumber() + outputFile = `database-${latestBlockNumber}.sqlite` + } + + const db: DB = await SQLiteConnector.create(schema, outputFile) + return FullNode.new({ + address: zkopruAddress, + provider, + db + }) +} diff --git a/src/zkopru-data.ts b/src/zkopru-data.ts new file mode 100644 index 0000000..30780ff --- /dev/null +++ b/src/zkopru-data.ts @@ -0,0 +1,105 @@ +import { Block as BlockCore } from '../zkopru/packages/core' +// import { Block as OriginBlock } from '../zkopru/packages/client/src/types' +import { FullNode } from '../zkopru/packages/core' + + +// Prefix e means Extend +// TODO: consider using this TreeNode +// interface TreeNode extends OriginTreeNode { +// utxoType: number // Deposit, Tx, Withdrawal +// } + +// interface Block extends OriginBlock { +// slash?: Slash +// } + +interface L2blockHashes { + [CanonicalBlockNumber: number]: string[] // Block Hash +} + +class ZkopruData { + fullNode: FullNode + + L2blockHashes: L2blockHashes // Fast + + lastUpdatedBlockCount: number + + constructor(fullNode: FullNode) { + this.fullNode = fullNode + this.L2blockHashes = {} + this.lastUpdatedBlockCount = -1 + } + + // Update state this data class every event recieved on full node + async updateL2BlockHashes() { + const { db } = this.fullNode + const totalProposalCount = await db.count('Proposal', { + include: { block: true } + }) + + // update 'L2blocks' double size of updateWindow + const updateBlockNum = totalProposalCount - this.lastUpdatedBlockCount + const updateWindow = Math.max(this.lastUpdatedBlockCount - updateBlockNum, 0) + + if (updateWindow > 0) { + // const targetProposals: {proposalNum: number}[] = [] + // for (const num of [...Array(updateWindow).keys()]) { + // targetProposals.push({proposalNum: startBlockNum + num}) + // } + const updatingProposals = await db.findMany('Proposal', { + where: {}, + orderBy: { proposalNum: 'desc' }, + limit: updateWindow + }) + + // update L2blocks data + for (const proposal of updatingProposals) { + const l2block = this.L2blockHashes[proposal.canonicalNum] + if (!l2block) { + this.L2blockHashes[proposal.canonicalNum] = [proposal.hash] + this.lastUpdatedBlockCount++ + continue + } + if (!l2block.includes(proposal.hash)) { + l2block.push(proposal.hash) + this.lastUpdatedBlockCount++ + } + } + } + } + + private getBlockHashesByNumber(CanonicalBlockNumber: number) { + if (this.L2blockHashes[CanonicalBlockNumber]) return undefined + return this.L2blockHashes[CanonicalBlockNumber] + } + + // TODO: full data export + // type from api-client + async getBlockData(CanonicalBlockNumber: number) { + const { db } = this.fullNode + const targetHashes = this.getBlockHashesByNumber(CanonicalBlockNumber) + + let blocks: any[] = [] + if (targetHashes && targetHashes.length > 0) { + for (const blockHash of targetHashes) { + // get data from database + // TODO: get Include related + const [ proposal, header, slash ] = await Promise.all([ + db.findOne('Proposal', { where: { hash: blockHash } }), + db.findOne('Header', { where: { hash: blockHash } }), + db.findOne('Slash', { where: { hash: blockHash } }) + ]) + // db.findOne(`Block`, { where: { hash: blockhash }, include: { block: true, }) + + // body parsing + const { proposalData, ...remains } = proposal + const blockData = BlockCore.from(proposalData) + blocks.push({ ...remains, header, blockData, slash}) + } + } + + return JSON.stringify(blocks) + } +} + +export default ZkopruData