diff --git a/.gitignore b/.gitignore index cc77112e..66190ff3 100644 --- a/.gitignore +++ b/.gitignore @@ -55,3 +55,6 @@ coverage.txt # conduit example cmd/conduit/data + +# examples +examples/blocks diff --git a/conduit/plugins/importers/algod/algod_importer_test.go b/conduit/plugins/importers/algod/algod_importer_test.go index 66a6b51e..5ff1696e 100644 --- a/conduit/plugins/importers/algod/algod_importer_test.go +++ b/conduit/plugins/importers/algod/algod_importer_test.go @@ -334,6 +334,29 @@ netaddr: %s assert.EqualError(t, err, fmt.Sprintf("algod importer was set to a mode (%s) that wasn't supported", name)) } +func TestInitParseUrlFailure(t *testing.T) { + url := ".0.0.0.0.0.0.0:1234" + testImporter := New() + cfgStr := fmt.Sprintf(`--- +mode: %s +netaddr: %s +`, "follower", url) + _, err := testImporter.Init(ctx, conduit.MakePipelineInitProvider(&pRound, nil), plugins.MakePluginConfig(cfgStr), logger) + assert.ErrorContains(t, err, "parse") +} + +func TestInitModeFailure(t *testing.T) { + name := "foobar" + ts := NewAlgodServer(GenesisResponder) + testImporter := New() + cfgStr := fmt.Sprintf(`--- +mode: %s +netaddr: %s +`, name, ts.URL) + _, err := testImporter.Init(ctx, conduit.MakePipelineInitProvider(&pRound, nil), plugins.MakePluginConfig(cfgStr), logger) + assert.EqualError(t, err, fmt.Sprintf("algod importer was set to a mode (%s) that wasn't supported", name)) +} + func TestInitGenesisFailure(t *testing.T) { t.Parallel() ctx := context.Background() diff --git a/examples/Justfile b/examples/Justfile new file mode 100644 index 00000000..6f2e04fb --- /dev/null +++ b/examples/Justfile @@ -0,0 +1,480 @@ +set export +set shell := ["zsh", "-cu"] + +NETWORKS := `echo $HOME` + "/networks" +NAME := "niftynetwork" +CURR_NETWORK := NETWORKS + "/" + NAME +GO_ALGORAND := "/Users/zeph/github/algorand/go-algorand" +NODE_TEMPLATE := GO_ALGORAND + "/test/testdata/nettemplates/TwoNodesFollower100Second.json" +PRIVATE_DATA_NODE := "Primary" +DATA_NODE := env_var_or_default("DATA_NODE", PRIVATE_DATA_NODE) +IS_PUBLIC_TRUTHY := env_var_or_default("DATA_NODE", "") +ALGORAND_DATA := CURR_NETWORK + "/" + DATA_NODE +ALGORAND_TOKEN_PATH := ALGORAND_DATA + "/algod.token" +ALGORAND_ADTOKEN_PATH := ALGORAND_DATA + "/algod.admin.token" +ALGORAND_ALGOD_PATH := ALGORAND_DATA + "/algod.net" +ALGORAND_PID_PATH := ALGORAND_DATA + "/algod.pid" + +# These fake pre-set tokens make it easier to test against a local network +PRESET_ALGOD_TOKEN := "16b29a0a2bbcc535f1e9e40f0c0888013f3789bf2bd34e7907c8fb1ae9d16024" +PRESET_ALGOD_ADMIN := "20064faacad1e590e757ac9492506c2d948633d7c458651b16a3991d26997695" + +# Older: +BOXES_TEAL := "boxes.teal" + +# --- SUMMARY --- # + +# list all available commands +default: + just --list + +# echo all variables +@echo: + echo NETWORKS: $NETWORKS + echo NAME: $NAME + echo CURR_NETWORK: $CURR_NETWORK + echo GO_ALGORAND: $GO_ALGORAND + echo NODE_TEMPLATE: $NODE_TEMPLATE + echo PRIVATE_DATA_NODE: $PRIVATE_DATA_NODE + echo DATA_NODE: $DATA_NODE + echo IS_PUBLIC_TRUTHY: $IS_PUBLIC_TRUTHY + echo ALGORAND_DATA: $ALGORAND_DATA + echo ALGORAND_TOKEN_PATH: $ALGORAND_TOKEN_PATH + echo ALGORAND_ALGOD_PATH: $ALGORAND_ALGOD_PATH + echo ALGORAND_PID_PATH: $ALGORAND_PID_PATH + + echo BOXES_TEAL: $BOXES_TEAL + +# --- algod curl --- # +algod ENDPOINT="/v2/status" VERB="GET": + #! /usr/bin/env bash + # set -euxo pipefail + set -euo pipefail + ALGORAND_TOKEN=$(cat ${ALGORAND_TOKEN_PATH}) + ALGORAND_ALGOD=$(cat ${ALGORAND_ALGOD_PATH}) + ALGORAND_PID=$(cat ${ALGORAND_PID_PATH}) + curl -X ${VERB} "http://${ALGORAND_ALGOD}${ENDPOINT}" -H "Authorization: Bearer ${ALGORAND_TOKEN}" + +# pass thru goal command but with the $ALGORAND_DATA set +goal *ARGS: + goal {{ARGS}} + +# --- GENERATOR SCRIPT COMMANDS --- # + +# generate an arbitrary number of app and box scenarios, each with up to BOXES_PER_APP boxes +gen-mult-app-boxes NUM_APPS="10" BOXES_PER_APP="2048": + #!/usr/bin/env python3 + import subprocess + from subprocess import CalledProcessError + + num_apps = int({{NUM_APPS}}) + print(f"{num_apps=}") + for i in range(num_apps): + print("\n", "\n", "\n", f"gen-app-and-box-scenarios #{i+1}" ) + subprocess.run(["just", "gen-app-and-box-scenarios", "{{BOXES_PER_APP}}"]).check_returncode() + + +# create an app and add up to BOXES_PER_APP random boxes to it in a multi-threaded fashion +gen-app-and-box-scenarios BOXES_PER_APP="10": + #!/usr/bin/env python3 + from concurrent.futures import ThreadPoolExecutor + import json + import logging + import random + import string + import subprocess + from subprocess import CalledProcessError + import time + + CHARS = string.digits + string.ascii_letters + VAL_SIZE = 24 + BOXES_PER_APP = int({{BOXES_PER_APP}}) + NLS = "\n" * 3 + + subprocess.run(["just", "app-create_fund"]).check_returncode() + + def worker(thread_number): + logging.info(f"HELLO from {thread_number}!") + + create_cpe = set_cpe = test_cpe = del_cpe = None + + rand_key_size = random.randint(4, 64) + rand_key = "".join(random.choice(CHARS) for _ in range(rand_key_size)) + print(f"{NLS}{thread_number}: {rand_key=}") + try: + subprocess.run(["just", "box-create", rand_key]).check_returncode() + except CalledProcessError as cpe: + create_cpe = str(cpe) + + rand_val = "".join(random.choice(CHARS) for _ in range(VAL_SIZE)) + print(f"{NLS}{thread_number}: {rand_val=}") + try: + subprocess.run(["just", "box-set", rand_key, rand_val]).check_returncode() + except CalledProcessError as cpe: + set_cpe = str(cpe) + + print(f"{NLS}{thread_number}: checking {rand_val=}") + try: + subprocess.run(["just", "box-test", rand_key, rand_val]).check_returncode() + except CalledProcessError as cpe: + test_cpe = str(cpe) + + delete = random.choice([True, False]) + if delete: + print(f"{NLS}{thread_number}: deleting") + try: + subprocess.run(["just", "box-delete", rand_key]).check_returncode() + except CalledProcessError as cpe: + del_cpe = str(cpe) + + return { + "thread_number": thread_number, + "key_size": rand_key_size, + "key": rand_key, + "val": rand_val, + "deleted": delete, + "called_process_errors": { + "create_cpe": create_cpe, + "set_cpe": set_cpe, + "test_cpe": test_cpe, + "del_cpe": del_cpe, + }, + } + + format = "%(asctime)s: %(message)s" + logging.basicConfig(format=format, level=logging.INFO, + datefmt="%H:%M:%S") + + results = [] + with ThreadPoolExecutor() as executor: + for r in executor.map(worker, range(BOXES_PER_APP)): + results.append(r) + + print(json.dumps(results, indent=2)) + + for result in results: + for err in result["called_process_errors"].values(): + if err: + raise err + +# --- HIGHER LEVEL --- # + +# create and then start (error if already created) +@create_and_start: create start status + sleep 5 + just status + +# create an app and then fund it +@app-create_fund: app-create last-app-fund + + +# --- BOX PUT: HIGHER LEVEL --- # + +# create box[BOX] for last app with provided key variable BOX +@box-create $BOX: + just app-call-last '\"create\", \"{{BOX}}\"' + +# set box[BOX]=VAL for last app with key BOX and val VAL +@box-set $BOX $VAL: + just app-call-last '\"set\", \"{{BOX}}\", \"{{VAL}}\"' + +# set box[BOX]=VAL for last app with key BOX and val VAL +@box-test $BOX $VAL: + just app-call-last '\"check\", \"{{BOX}}\", \"{{VAL}}\"' + +# delete box[BOX] for last app +@box-delete $BOX: + just app-call-last '\"delete\", \"{{BOX}}\"' + +# stop and tear down the node network. WARNING: YOU WILL LOSE ALL YOUR NODE DATA FROM THE FILE SYSTEM. +@stop_and_nuke: stop nuke + +# --- PRE-REQUISITES --- # + +# calculate an app's address using the python SDK +app-address *ARGS: + #!/usr/bin/env python3 + from algosdk import logic + print(logic.get_application_address({{ ARGS }})) + +# --- NETWORKS / NODES --- # + +# Private vs. Public Networks. Typical workflow: +# 1. Create a directory for your network (CURR_NETWORK = NETWORKS/NAME) +# 2. Populate the node information under CURR_NETWORK. Branch on Public vs. Private. +# In either case the the data directory is ALGORAND_DATA == CURR_NETWORK/DATA_NODE == NETWORKS/NAME/DATA_NODE. +# a. Public: Use algocfg to configure a single node: see `just pub-create` +# b. Private: Use `goal` to configure a network nodes under CURR_NETWORK. See `just create` +# 3. Start the network: `just start` +# +# NOTE: To run a public network commands, you need to supply the env var `DATA_NODE`. EG: +# DATA_NODE=Follower just pub-validate-datadir +# Or, export and run: +# export DATA_NODE=Follower +# just pub-validate-datadir + + +# create a private network with one node (error if already created) +create: + mkdir -p $NETWORKS + goal network create -n $NAME -r $CURR_NETWORK -t $NODE_TEMPLATE + +# print out the current network's data directory tree +@tree: + tree $CURR_NETWORK + +# start a the network (error if already running or not created) +@start: + goal node start + + +# PUBLIC NETWORKS BEGIN + +# check that is ready to connect to public network +pub-validate-datadir: + #! /usr/bin/env bash + set -euxo pipefail + [ -z "$IS_PUBLIC_TRUTHY" ] && { echo "Error: DATA_NODE env var required for public network" ; exit 1; } + echo "Ready for public network with node datadir: $ALGORAND_DATA" + +# list available profiles for configuring a network +@pub-cfg-list: + algocfg profile list + +# show the current network configuration +@pub-cfg-show: + echo "cat ${ALGORAND_DATA}/config.json" + cat ${ALGORAND_DATA}/config.json + +# prepare for connecting to public network +pub-prepare: pub-validate-datadir + mkdir -p $ALGORAND_DATA + +# configure a Conduit's network using `algocfg` +pub-create NODE_PROFILE="conduit" NETWORK="testnet" ENDPOINT="127.0.0.1:56765" PT="1" PAT="1": pub-prepare + algocfg profile set {{NODE_PROFILE}} -d $ALGORAND_DATA + [ -n {{ENDPOINT}} ] && echo "setting ENDPOINT={{ENDPOINT}}" && algocfg set -p EndpointAddress -v {{ENDPOINT}} + [ {{PT}} = "1" ] && echo "setting ALGOD TOKEN=${PRESET_ALGOD_TOKEN}" && echo ${PRESET_ALGOD_TOKEN} > ${ALGORAND_TOKEN_PATH} + [ {{PAT}} = "1" ] && echo "setting ALGOD ADMIN TOKEN=${PRESET_ALGOD_ADMIN}" && echo ${PRESET_ALGOD_ADMIN} > ${ALGORAND_ADTOKEN_PATH} + cp ${GO_ALGORAND}/installer/genesis/{{NETWORK}}/genesis.json $ALGORAND_DATA + +# status of network node +@status: + goal node status && echo "RUNNING" || echo "NOT RUNNING" + +# stop the running node (error if not running) +@stop: + goal node stop + +# remove the node's data from the file system +@nuke: + echo "deleting $CURR_NETWORK" + rm -rf $CURR_NETWORK + +# --- ACCOUNTS --- # + +# list all associated accounts +@list: + goal account list + +# create a new account without renaming it to a human friendly local alias +@raw-new-account: + goal account new | awk '{print $NF}' + +# echo an account's alias +@account-alias $ACCOUNT: + just list | grep {{ACCOUNT}} | awk '{print $2}' + +# create a new locally aliased account +@new-account $ALIAS $ACCOUNT=`just raw-new-account`: + goal account rename `just account-alias {{ACCOUNT}}` {{ALIAS}} + +# create a new multisig account with threshold 1 using provided accounts (cannot handle aliases) +@raw-msig-account *ACCOUNTS: + goal account multisig new -T 1 {{ACCOUNTS}} | awk '{print $NF}' + +# create a new multisig account with given ALIAS and threshold 1 using provided accounts (cannot handle aliases) +@new-msig-account $ALIAS *ACCOUNTS: + goal account rename `just account-alias $(just raw-msig-account {{ACCOUNTS}})` {{ALIAS}} + +# funding account's address +@funder: + just list | awk '{print $2}' + +# provide information about a given account +@info $ACCOUNT=`just funder`: + goal account info --address {{ACCOUNT}} + +# provide an account's balance +@balance $ACCOUNT=`just funder`: + goal account balance --address {{ACCOUNT}} + +# funder's most recently created app-id +@last-app-id: + just info | grep ID | tail -n 1 | cut -d "," -f1 | awk '{print $2}' + +# the account address of the funders most recently created app-id +@last-app-address: + just app-address `just last-app-id` + +# --- ASSETS --- # + +# create a dummy asset for the provided FUNDER. Copy pasta from: https://dappradar.com/blog/algorand-dapp-development-2-standard-asset-management +@asset-create $FUNDER=`just funder`: + goal asset create --creator {{FUNDER}} --total 1000000 --unitname bUSD --name "Balgorand USD" --asseturl "https://b-usd.com" --decimals 9 + +# --- APPLICATIONS --- # + +# information about an application of given id +@app-info $APP_ID=`just last-app-id`: + goal app info --app-id {{APP_ID}} + +# print out the boxes teal program +@boxes_teal: + cat $BOXES_TEAL + +# shortcut for the approval and clear program `goal app create` params +@programs: + echo "--approval-prog $BOXES_TEAL --clear-prog clear.teal" + +# shortcut for the storage params of `goal app create` +@app-vars $GBS="0" $GI="0" $LBS="0" $LI="0": + echo "--global-byteslices $GBS --global-ints $GI --local-byteslices $LBS --local-ints $LI" + +# shortcut for creating the arguments of an app call +box-app-args *ARGS: + #!/usr/bin/env python3 + args = [] + box_arg = "" + i = 0 + for arg in {{ARGS}}: + try: + int(arg) + arg = f"int:{arg}" + except Exception: + arg = f"str:{arg}" + args.extend(["--app-arg", arg]) + if i == 1: + box_arg = f"--box {arg}" + i += 1 + if box_arg: + args = [box_arg] + args + print(*args) + +# create an app funded by funder account +@app-create $GBS="0" $GI="0" $LBS="0" $LI="0": + echo "goal app create --creator `just funder` `just programs` `just app-vars $GBS $GI $LBS $LI`" + goal app create --creator `just funder` `just programs` `just app-vars $GBS $GI $LBS $LI` + +# call the last app from the funder address using ARGS +@app-call-last *ARGS='\"create\", \"mybox\"': + (set -x; goal app call --app-id `just last-app-id` --from `just funder` `just box-app-args {{ARGS}}`) + +# --- BOX INFO --- # + +# get all the boxes associated a given app-id +app-box-list $APP_ID=`just last-app-id`: + goal app box list --app-id {{APP_ID}} --max 0 + +# get box information for a given app-id and box name +app-box-info $APP_ID=`just last-app-id` $BOX="str:mybox": + goal app box info --app-id {{APP_ID}} --name {{BOX}} + + +# --- CLERK --- # + +# send from one account to another a given amount +@send $FROM $TO $AMOUNT: + goal clerk send --from {{FROM}} --to {{TO}} --amount {{AMOUNT}} + +# fund the most recently created app +@last-app-fund $AMOUNT=`echo 133713371337`: + just send `just funder` `just last-app-address` {{AMOUNT}} + +# --- CONSENSUS PARAMS --- # + +# list all consensus param declarations +@consensus-params-list: + cat ${GO_ALGORAND}/config/consensus.go | egrep " (bool|byte|int$|uint|map\[|Duration|PaysetCommitType)" | egrep -v "type|=|func|,|//" + +# print out the value history of a consensus param +@consensus-param $CP="MaximumMinimumBalance": + cat ${GO_ALGORAND}/config/consensus.go | grep {{CP}} || echo "{{CP}} not found in consensus.go" + +# consensus params for program size +consensus-prog-size: + just consensus-param LogicSigMaxSize + just consensus-param MaxAppProgramLen + +# consensus param for LogicSigMaxCost +consensus-prog-cost: + just consensus-param MaxCost + just consensus-param MaxAppProgramCost + +# consensus params for program pages +@consensus-prog-pages: + just consensus-param MaxExtraAppProgramPages + +# consensus params for foreign refs: +consensus-foreign-refs: + just consensus-param MaxAppTxnAccounts + just consensus-param MaxAppTxnForeignApps + just consensus-param MaxAppTxnForeignAssets + just consensus-param MaxAppTotalTxnReferences + just consensus-param MaxAppBoxReferences + +# consensus params for local/global storage: +consensus-storage: + just consensus-param MaxAppKeyLen + just consensus-param MaxAppBytesValueLen + just consensus-param MaxAppSumKeyValueLens + just consensus-param MaxLocalSchemaEntries + just consensus-param MaxGlobalSchemaEntries + +# consensus params for min-balance calc: +consensus-minbal: + just consensus-param SchemaMinBalancePerEntry + just consensus-param SchemaUintMinBalance + just consensus-param SchemaBytesMinBalance + just consensus-param BoxFlatMinBalance + just consensus-param BoxByteMinBalance + +# consensus params for boxes: +consensus-boxes: + just consensus-param MaxAppKeyLen + just consensus-param MaxBoxSize + just consensus-param BoxFlatMinBalance + just consensus-param BoxByteMinBalance + just consensus-param MaxAppBoxReferences + just consensus-param BytesPerBoxReference + +# --- MISCELLANEOUS --- # + +# print out the network's algod & kmd token and network/process info +client-info: + #! /usr/bin/env bash + set -euo pipefail + ALGORAND_TOKEN=$(cat ${ALGORAND_TOKEN_PATH}) + ALGORAND_ADTOKEN=$(cat ${ALGORAND_ADTOKEN_PATH}) + ALGORAND_ALGOD=$(cat ${ALGORAND_ALGOD_PATH}) + ALGORAND_PID=$(cat ${ALGORAND_PID_PATH}) + echo "algod.token: ${ALGORAND_TOKEN}" + echo "algod.admin.token: ${ALGORAND_ADTOKEN}" + echo "algod.net: ${ALGORAND_ALGOD}" + echo "algod.pid: ${ALGORAND_PID}" + + if [[ -f "${ALGORAND_DATA}/kmd-v0.5/kmd.token" ]]; then + KMD=$(cat "${ALGORAND_DATA}/kmd-v0.5/kmd.token") + else + KMD="" + fi + echo "kmd.token: ${KMD}" + if [[ -n $KMD ]]; then + echo "kmd.net---->" + cat ${ALGORAND_DATA}/kmd-v0.5/kmd.log | grep 127.0.0.1 | head -n 1 | cut -d '"' -f4 + fi + +# print out broadcastQueueBulk's channel size ... the default is 100 which is too small for the example +@broadcast-queue-size: + echo "default is 100. What is it actually in network/wsNetwork.go ?" + cat {{GO_ALGORAND}}/network/wsNetwork.go | grep "wn.broadcastQueueBulk = make(chan broadcastRequest" | cut -d "," -f2 | cut -d ")" -f1 | awk '{print $1}' \ No newline at end of file diff --git a/examples/Justfile.tmpl b/examples/Justfile.tmpl new file mode 100644 index 00000000..a8fd7a01 --- /dev/null +++ b/examples/Justfile.tmpl @@ -0,0 +1,480 @@ +set export +set shell := ["zsh", "-cu"] + +NETWORKS := $${NETWORKS} # parent directory of all networks +NAME := "$${NAME}" +CURR_NETWORK := NETWORKS + "/" + NAME +GO_ALGORAND := "$${GO_ALGORAND}" +NODE_TEMPLATE := GO_ALGORAND + "/test/testdata/nettemplates/$${NODE_TEMPLATE}" +PRIVATE_DATA_NODE := "$${PRIVATE_DATA_NODE}" +DATA_NODE := env_var_or_default("DATA_NODE", PRIVATE_DATA_NODE) +IS_PUBLIC_TRUTHY := env_var_or_default("DATA_NODE", "") +ALGORAND_DATA := CURR_NETWORK + "/" + DATA_NODE +ALGORAND_TOKEN_PATH := ALGORAND_DATA + "/algod.token" +ALGORAND_ADTOKEN_PATH := ALGORAND_DATA + "/algod.admin.token" +ALGORAND_ALGOD_PATH := ALGORAND_DATA + "/algod.net" +ALGORAND_PID_PATH := ALGORAND_DATA + "/algod.pid" + +# These fake pre-set tokens make it easier to test against a local network +PRESET_ALGOD_TOKEN := "16b29a0a2bbcc535f1e9e40f0c0888013f3789bf2bd34e7907c8fb1ae9d16024" +PRESET_ALGOD_ADMIN := "20064faacad1e590e757ac9492506c2d948633d7c458651b16a3991d26997695" + +# Older: +BOXES_TEAL := "boxes.teal" + +# --- SUMMARY --- # + +# list all available commands +default: + just --list + +# echo all variables +@echo: + echo NETWORKS: $NETWORKS + echo NAME: $NAME + echo CURR_NETWORK: $CURR_NETWORK + echo GO_ALGORAND: $GO_ALGORAND + echo NODE_TEMPLATE: $NODE_TEMPLATE + echo PRIVATE_DATA_NODE: $PRIVATE_DATA_NODE + echo DATA_NODE: $DATA_NODE + echo IS_PUBLIC_TRUTHY: $IS_PUBLIC_TRUTHY + echo ALGORAND_DATA: $ALGORAND_DATA + echo ALGORAND_TOKEN_PATH: $ALGORAND_TOKEN_PATH + echo ALGORAND_ALGOD_PATH: $ALGORAND_ALGOD_PATH + echo ALGORAND_PID_PATH: $ALGORAND_PID_PATH + + echo BOXES_TEAL: $BOXES_TEAL + +# --- algod curl --- # +algod ENDPOINT="/v2/status" VERB="GET": + #! /usr/bin/env bash + # set -euxo pipefail + set -euo pipefail + ALGORAND_TOKEN=$(cat ${ALGORAND_TOKEN_PATH}) + ALGORAND_ALGOD=$(cat ${ALGORAND_ALGOD_PATH}) + ALGORAND_PID=$(cat ${ALGORAND_PID_PATH}) + curl -X ${VERB} "http://${ALGORAND_ALGOD}${ENDPOINT}" -H "Authorization: Bearer ${ALGORAND_TOKEN}" + +# pass thru goal command but with the $ALGORAND_DATA set +goal *ARGS: + goal {{ARGS}} + +# --- GENERATOR SCRIPT COMMANDS --- # + +# generate an arbitrary number of app and box scenarios, each with up to BOXES_PER_APP boxes +gen-mult-app-boxes NUM_APPS="10" BOXES_PER_APP="2048": + #!/usr/bin/env python3 + import subprocess + from subprocess import CalledProcessError + + num_apps = int({{NUM_APPS}}) + print(f"{num_apps=}") + for i in range(num_apps): + print("\n", "\n", "\n", f"gen-app-and-box-scenarios #{i+1}" ) + subprocess.run(["just", "gen-app-and-box-scenarios", "{{BOXES_PER_APP}}"]).check_returncode() + + +# create an app and add up to BOXES_PER_APP random boxes to it in a multi-threaded fashion +gen-app-and-box-scenarios BOXES_PER_APP="10": + #!/usr/bin/env python3 + from concurrent.futures import ThreadPoolExecutor + import json + import logging + import random + import string + import subprocess + from subprocess import CalledProcessError + import time + + CHARS = string.digits + string.ascii_letters + VAL_SIZE = 24 + BOXES_PER_APP = int({{BOXES_PER_APP}}) + NLS = "\n" * 3 + + subprocess.run(["just", "app-create_fund"]).check_returncode() + + def worker(thread_number): + logging.info(f"HELLO from {thread_number}!") + + create_cpe = set_cpe = test_cpe = del_cpe = None + + rand_key_size = random.randint(4, 64) + rand_key = "".join(random.choice(CHARS) for _ in range(rand_key_size)) + print(f"{NLS}{thread_number}: {rand_key=}") + try: + subprocess.run(["just", "box-create", rand_key]).check_returncode() + except CalledProcessError as cpe: + create_cpe = str(cpe) + + rand_val = "".join(random.choice(CHARS) for _ in range(VAL_SIZE)) + print(f"{NLS}{thread_number}: {rand_val=}") + try: + subprocess.run(["just", "box-set", rand_key, rand_val]).check_returncode() + except CalledProcessError as cpe: + set_cpe = str(cpe) + + print(f"{NLS}{thread_number}: checking {rand_val=}") + try: + subprocess.run(["just", "box-test", rand_key, rand_val]).check_returncode() + except CalledProcessError as cpe: + test_cpe = str(cpe) + + delete = random.choice([True, False]) + if delete: + print(f"{NLS}{thread_number}: deleting") + try: + subprocess.run(["just", "box-delete", rand_key]).check_returncode() + except CalledProcessError as cpe: + del_cpe = str(cpe) + + return { + "thread_number": thread_number, + "key_size": rand_key_size, + "key": rand_key, + "val": rand_val, + "deleted": delete, + "called_process_errors": { + "create_cpe": create_cpe, + "set_cpe": set_cpe, + "test_cpe": test_cpe, + "del_cpe": del_cpe, + }, + } + + format = "%(asctime)s: %(message)s" + logging.basicConfig(format=format, level=logging.INFO, + datefmt="%H:%M:%S") + + results = [] + with ThreadPoolExecutor() as executor: + for r in executor.map(worker, range(BOXES_PER_APP)): + results.append(r) + + print(json.dumps(results, indent=2)) + + for result in results: + for err in result["called_process_errors"].values(): + if err: + raise err + +# --- HIGHER LEVEL --- # + +# create and then start (error if already created) +@create_and_start: create start status + sleep 5 + just status + +# create an app and then fund it +@app-create_fund: app-create last-app-fund + + +# --- BOX PUT: HIGHER LEVEL --- # + +# create box[BOX] for last app with provided key variable BOX +@box-create $BOX: + just app-call-last '\"create\", \"{{BOX}}\"' + +# set box[BOX]=VAL for last app with key BOX and val VAL +@box-set $BOX $VAL: + just app-call-last '\"set\", \"{{BOX}}\", \"{{VAL}}\"' + +# set box[BOX]=VAL for last app with key BOX and val VAL +@box-test $BOX $VAL: + just app-call-last '\"check\", \"{{BOX}}\", \"{{VAL}}\"' + +# delete box[BOX] for last app +@box-delete $BOX: + just app-call-last '\"delete\", \"{{BOX}}\"' + +# stop and tear down the node network. WARNING: YOU WILL LOSE ALL YOUR NODE DATA FROM THE FILE SYSTEM. +@stop_and_nuke: stop nuke + +# --- PRE-REQUISITES --- # + +# calculate an app's address using the python SDK +app-address *ARGS: + #!/usr/bin/env python3 + from algosdk import logic + print(logic.get_application_address({{ ARGS }})) + +# --- NETWORKS / NODES --- # + +# Private vs. Public Networks. Typical workflow: +# 1. Create a directory for your network (CURR_NETWORK = NETWORKS/NAME) +# 2. Populate the node information under CURR_NETWORK. Branch on Public vs. Private. +# In either case the the data directory is ALGORAND_DATA == CURR_NETWORK/DATA_NODE == NETWORKS/NAME/DATA_NODE. +# a. Public: Use algocfg to configure a single node: see `just pub-create` +# b. Private: Use `goal` to configure a network nodes under CURR_NETWORK. See `just create` +# 3. Start the network: `just start` +# +# NOTE: To run a public network commands, you need to supply the env var `DATA_NODE`. EG: +# DATA_NODE=Follower just pub-validate-datadir +# Or, export and run: +# export DATA_NODE=Follower +# just pub-validate-datadir + + +# create a private network with one node (error if already created) +create: + mkdir -p $NETWORKS + goal network create -n $NAME -r $CURR_NETWORK -t $NODE_TEMPLATE + +# print out the current network's data directory tree +@tree: + tree $CURR_NETWORK + +# start a the network (error if already running or not created) +@start: + goal node start + + +# PUBLIC NETWORKS BEGIN + +# check that is ready to connect to public network +pub-validate-datadir: + #! /usr/bin/env bash + set -euxo pipefail + [ -z "$IS_PUBLIC_TRUTHY" ] && { echo "Error: DATA_NODE env var required for public network" ; exit 1; } + echo "Ready for public network with node datadir: $ALGORAND_DATA" + +# list available profiles for configuring a network +@pub-cfg-list: + algocfg profile list + +# show the current network configuration +@pub-cfg-show: + echo "cat ${ALGORAND_DATA}/config.json" + cat ${ALGORAND_DATA}/config.json + +# prepare for connecting to public network +pub-prepare: pub-validate-datadir + mkdir -p $ALGORAND_DATA + +# configure a Conduit's network using `algocfg` +pub-create NODE_PROFILE="conduit" NETWORK="testnet" ENDPOINT="127.0.0.1:56765" PT="1" PAT="1": pub-prepare + algocfg profile set {{NODE_PROFILE}} -d $ALGORAND_DATA + [ -n {{ENDPOINT}} ] && echo "setting ENDPOINT={{ENDPOINT}}" && algocfg set -p EndpointAddress -v {{ENDPOINT}} + [ {{PT}} = "1" ] && echo "setting ALGOD TOKEN=${PRESET_ALGOD_TOKEN}" && echo ${PRESET_ALGOD_TOKEN} > ${ALGORAND_TOKEN_PATH} + [ {{PAT}} = "1" ] && echo "setting ALGOD ADMIN TOKEN=${PRESET_ALGOD_ADMIN}" && echo ${PRESET_ALGOD_ADMIN} > ${ALGORAND_ADTOKEN_PATH} + cp ${GO_ALGORAND}/installer/genesis/{{NETWORK}}/genesis.json $ALGORAND_DATA + +# status of network node +@status: + goal node status && echo "RUNNING" || echo "NOT RUNNING" + +# stop the running node (error if not running) +@stop: + goal node stop + +# remove the node's data from the file system +@nuke: + echo "deleting $CURR_NETWORK" + rm -rf $CURR_NETWORK + +# --- ACCOUNTS --- # + +# list all associated accounts +@list: + goal account list + +# create a new account without renaming it to a human friendly local alias +@raw-new-account: + goal account new | awk '{print $NF}' + +# echo an account's alias +@account-alias $ACCOUNT: + just list | grep {{ACCOUNT}} | awk '{print $2}' + +# create a new locally aliased account +@new-account $ALIAS $ACCOUNT=`just raw-new-account`: + goal account rename `just account-alias {{ACCOUNT}}` {{ALIAS}} + +# create a new multisig account with threshold 1 using provided accounts (cannot handle aliases) +@raw-msig-account *ACCOUNTS: + goal account multisig new -T 1 {{ACCOUNTS}} | awk '{print $NF}' + +# create a new multisig account with given ALIAS and threshold 1 using provided accounts (cannot handle aliases) +@new-msig-account $ALIAS *ACCOUNTS: + goal account rename `just account-alias $(just raw-msig-account {{ACCOUNTS}})` {{ALIAS}} + +# funding account's address +@funder: + just list | awk '{print $2}' + +# provide information about a given account +@info $ACCOUNT=`just funder`: + goal account info --address {{ACCOUNT}} + +# provide an account's balance +@balance $ACCOUNT=`just funder`: + goal account balance --address {{ACCOUNT}} + +# funder's most recently created app-id +@last-app-id: + just info | grep ID | tail -n 1 | cut -d "," -f1 | awk '{print $2}' + +# the account address of the funders most recently created app-id +@last-app-address: + just app-address `just last-app-id` + +# --- ASSETS --- # + +# create a dummy asset for the provided FUNDER. Copy pasta from: https://dappradar.com/blog/algorand-dapp-development-2-standard-asset-management +@asset-create $FUNDER=`just funder`: + goal asset create --creator {{FUNDER}} --total 1000000 --unitname bUSD --name "Balgorand USD" --asseturl "https://b-usd.com" --decimals 9 + +# --- APPLICATIONS --- # + +# information about an application of given id +@app-info $APP_ID=`just last-app-id`: + goal app info --app-id {{APP_ID}} + +# print out the boxes teal program +@boxes_teal: + cat $BOXES_TEAL + +# shortcut for the approval and clear program `goal app create` params +@programs: + echo "--approval-prog $BOXES_TEAL --clear-prog clear.teal" + +# shortcut for the storage params of `goal app create` +@app-vars $GBS="0" $GI="0" $LBS="0" $LI="0": + echo "--global-byteslices $GBS --global-ints $GI --local-byteslices $LBS --local-ints $LI" + +# shortcut for creating the arguments of an app call +box-app-args *ARGS: + #!/usr/bin/env python3 + args = [] + box_arg = "" + i = 0 + for arg in {{ARGS}}: + try: + int(arg) + arg = f"int:{arg}" + except Exception: + arg = f"str:{arg}" + args.extend(["--app-arg", arg]) + if i == 1: + box_arg = f"--box {arg}" + i += 1 + if box_arg: + args = [box_arg] + args + print(*args) + +# create an app funded by funder account +@app-create $GBS="0" $GI="0" $LBS="0" $LI="0": + echo "goal app create --creator `just funder` `just programs` `just app-vars $GBS $GI $LBS $LI`" + goal app create --creator `just funder` `just programs` `just app-vars $GBS $GI $LBS $LI` + +# call the last app from the funder address using ARGS +@app-call-last *ARGS='\"create\", \"mybox\"': + (set -x; goal app call --app-id `just last-app-id` --from `just funder` `just box-app-args {{ARGS}}`) + +# --- BOX INFO --- # + +# get all the boxes associated a given app-id +app-box-list $APP_ID=`just last-app-id`: + goal app box list --app-id {{APP_ID}} --max 0 + +# get box information for a given app-id and box name +app-box-info $APP_ID=`just last-app-id` $BOX="str:mybox": + goal app box info --app-id {{APP_ID}} --name {{BOX}} + + +# --- CLERK --- # + +# send from one account to another a given amount +@send $FROM $TO $AMOUNT: + goal clerk send --from {{FROM}} --to {{TO}} --amount {{AMOUNT}} + +# fund the most recently created app +@last-app-fund $AMOUNT=`echo 133713371337`: + just send `just funder` `just last-app-address` {{AMOUNT}} + +# --- CONSENSUS PARAMS --- # + +# list all consensus param declarations +@consensus-params-list: + cat ${GO_ALGORAND}/config/consensus.go | egrep " (bool|byte|int$|uint|map\[|Duration|PaysetCommitType)" | egrep -v "type|=|func|,|//" + +# print out the value history of a consensus param +@consensus-param $CP="MaximumMinimumBalance": + cat ${GO_ALGORAND}/config/consensus.go | grep {{CP}} || echo "{{CP}} not found in consensus.go" + +# consensus params for program size +consensus-prog-size: + just consensus-param LogicSigMaxSize + just consensus-param MaxAppProgramLen + +# consensus param for LogicSigMaxCost +consensus-prog-cost: + just consensus-param MaxCost + just consensus-param MaxAppProgramCost + +# consensus params for program pages +@consensus-prog-pages: + just consensus-param MaxExtraAppProgramPages + +# consensus params for foreign refs: +consensus-foreign-refs: + just consensus-param MaxAppTxnAccounts + just consensus-param MaxAppTxnForeignApps + just consensus-param MaxAppTxnForeignAssets + just consensus-param MaxAppTotalTxnReferences + just consensus-param MaxAppBoxReferences + +# consensus params for local/global storage: +consensus-storage: + just consensus-param MaxAppKeyLen + just consensus-param MaxAppBytesValueLen + just consensus-param MaxAppSumKeyValueLens + just consensus-param MaxLocalSchemaEntries + just consensus-param MaxGlobalSchemaEntries + +# consensus params for min-balance calc: +consensus-minbal: + just consensus-param SchemaMinBalancePerEntry + just consensus-param SchemaUintMinBalance + just consensus-param SchemaBytesMinBalance + just consensus-param BoxFlatMinBalance + just consensus-param BoxByteMinBalance + +# consensus params for boxes: +consensus-boxes: + just consensus-param MaxAppKeyLen + just consensus-param MaxBoxSize + just consensus-param BoxFlatMinBalance + just consensus-param BoxByteMinBalance + just consensus-param MaxAppBoxReferences + just consensus-param BytesPerBoxReference + +# --- MISCELLANEOUS --- # + +# print out the network's algod & kmd token and network/process info +client-info: + #! /usr/bin/env bash + set -euo pipefail + ALGORAND_TOKEN=$(cat ${ALGORAND_TOKEN_PATH}) + ALGORAND_ADTOKEN=$(cat ${ALGORAND_ADTOKEN_PATH}) + ALGORAND_ALGOD=$(cat ${ALGORAND_ALGOD_PATH}) + ALGORAND_PID=$(cat ${ALGORAND_PID_PATH}) + echo "algod.token: ${ALGORAND_TOKEN}" + echo "algod.admin.token: ${ALGORAND_ADTOKEN}" + echo "algod.net: ${ALGORAND_ALGOD}" + echo "algod.pid: ${ALGORAND_PID}" + + if [[ -f "${ALGORAND_DATA}/kmd-v0.5/kmd.token" ]]; then + KMD=$(cat "${ALGORAND_DATA}/kmd-v0.5/kmd.token") + else + KMD="" + fi + echo "kmd.token: ${KMD}" + if [[ -n $KMD ]]; then + echo "kmd.net---->" + cat ${ALGORAND_DATA}/kmd-v0.5/kmd.log | grep 127.0.0.1 | head -n 1 | cut -d '"' -f4 + fi + +# print out broadcastQueueBulk's channel size ... the default is 100 which is too small for the example +@broadcast-queue-size: + echo "default is 100. What is it actually in network/wsNetwork.go ?" + cat {{GO_ALGORAND}}/network/wsNetwork.go | grep "wn.broadcastQueueBulk = make(chan broadcastRequest" | cut -d "," -f2 | cut -d ")" -f1 | awk '{print $1}' \ No newline at end of file diff --git a/examples/binary_polars.py b/examples/binary_polars.py new file mode 100644 index 00000000..deab3a79 --- /dev/null +++ b/examples/binary_polars.py @@ -0,0 +1,111 @@ +from dataclasses import dataclass +import json +import os +from pathlib import Path +import sys + +import polars as pl + +DEBUGGING = True + +STOP_SIGNAL = "!!!STOP!!!" + +EXAMPLES = Path.cwd() / "examples" +BLK_DIR = EXAMPLES / "blocks" +PLDB = EXAMPLES / "pldb" + +POLARS_TXNS = PLDB / "transactions.feather" +POLARS_BLKS = PLDB / "blocks.feather" + + +@dataclass +class PolarsDF: + file: Path + df: pl.LazyFrame | pl.DataFrame | None = None + + +@dataclass +class PolarsDB: + txns: PolarsDF + blks: PolarsDF + + +def setup(): + db = PolarsDB( + txns=PolarsDF(file=POLARS_TXNS), + blks=PolarsDF(file=POLARS_BLKS), + ) + + try: + db.blks.df = pl.scan_ipc(POLARS_BLKS) + db.txns.df = pl.scan_ipc(POLARS_TXNS) + except FileNotFoundError as fnfe: + print(f"File not found: {fnfe}") + + return db + + +import polars as pl + +# Assuming you have two Polars DataFrames `df1` and `df2` with the given structure +# Replace 'df1' and 'df2' with your actual DataFrame variable names + + +# Define a function to extract the 'rnd' field from the block dictionaries +def extract_rnd(blk: dict) -> int: + return blk["rnd"] + +def fix_missing_rnd(blk): + if "rnd" not in blk["block"]: + blk["block"]["rnd"] = 0 + return blk + +def merge(pdf: PolarsDF, blocks: list[dict]): + orig_len = 0 + if pdf.df is None: + pdf.df = pl.DataFrame([fix_missing_rnd(blk) for blk in blocks]) + else: + assert isinstance(pdf.df, pl.LazyFrame) + orig = pdf.df.collect() + orig_len = len(orig) + orig_rounds = set(orig["block"].apply(extract_rnd).to_list()) + + if blks := [ + fix_missing_rnd(blk) + for blk in blocks + if blk["block"].get("rnd", 0) not in orig_rounds + ]: + try: + pdf.df = orig.extend(pl.DataFrame(blks)) + except pl.ShapeError as se: + print(f"ShapeError: {se}") + bs, ds = (otd := orig.to_dict())['block'], otd['delta'] + collected_dicts = [{'block': b, 'delta': d} for b, d in zip(bs, ds)] + pdf.df = pl.DataFrame(collected_dicts + blks) + else: + print(f"No new blocks to add to {pdf.file}") + return + + pdf.df.write_ipc(pdf.file) + print( + f"{pdf.file} updated with {len(blocks)} blocks from original {orig_len} blocks" + ) + + +if __name__ == "__main__": + blocks_iter = sorted(os.listdir(BLK_DIR)) if DEBUGGING else sys.stdin + + db = setup() + blocks = [] + try: + for i, line in enumerate(blocks_iter): + if not (trimmed := line.strip()): + continue + + print(f"{i}. {trimmed=}") + with open(f"{BLK_DIR}/{trimmed}") as f: + blocks.append(json.loads(f.read())) + except KeyboardInterrupt: + print("TERMINATING PROGRAM") + finally: + merge(db.blks, blocks) diff --git a/examples/data/conduit.yml b/examples/data/conduit.yml new file mode 100644 index 00000000..225ff4a6 --- /dev/null +++ b/examples/data/conduit.yml @@ -0,0 +1,45 @@ +log-level: INFO + +# If no log file is provided logs are written to stdout. +#log-file: + +retry-count: 1 + +retry-delay: "1s" + +# Optional filepath to use for pidfile. +#pid-filepath: /path/to/pidfile + +# Whether or not to print the conduit banner on startup. +hide-banner: false + +# When enabled prometheus metrics are available on '/metrics' +metrics: + mode: OFF + addr: ":9999" + prefix: "conduit" + +importer: + name: algod + config: + mode: "follower" + netaddr: "http://127.0.0.1:56765" + token: "16b29a0a2bbcc535f1e9e40f0c0888013f3789bf2bd34e7907c8fb1ae9d16024" + catchup-config: + admin-token: "20064faacad1e590e757ac9492506c2d948633d7c458651b16a3991d26997695" + +processors: + - name: filter_processor + config: + filters: + - any: + - tag: txn.snd + expression-type: equal + expression: "NOTIFY2IJIXDNDCHF4EJZJOQYEQACIEVAIXBQMSZU4YTDKHWPLVYGTOU5Y" + +exporter: + name: "file_writer" + config: + block-dir: "/Users/zeph/github/algorand/conduit/examples/blocks" + filename-pattern: "block_%09d.json" + drop-certificate: true diff --git a/examples/make_just.py b/examples/make_just.py new file mode 100644 index 00000000..c62b6acf --- /dev/null +++ b/examples/make_just.py @@ -0,0 +1,48 @@ +import argparse +from string import Template + +# EG: +# +# ❯ python make_just.py --name blue-whale + + +class DblDollars(Template): + delimiter = '$$' # Use $$ as the delimiter + +# Define the substitutions +substitutions = { + "NAME": "niftynetwork", + "NETWORKS": '`echo $HOME` + "/networks"', + "GO_ALGORAND": "/Users/zeph/github/algorand/go-algorand", + "NODE_TEMPLATE": "OneNodeFuture.json", + "PRIVATE_DATA_NODE": "Primary", +} + +# Parse command-line arguments +parser = argparse.ArgumentParser() +parser.add_argument("--just-out-file", default="Justfile.tmp") +parser.add_argument("--name") +parser.add_argument("--networks") +parser.add_argument("--go-algorand") +parser.add_argument("--node-template") +parser.add_argument("--private-data-node") + + +args = parser.parse_args() +templ_args = vars(args) +file_out = templ_args.pop("just_out_file") + +# Update the substitutions with the arguments +substitutions.update({ + key: templ_args[k] for key in substitutions if templ_args[(k := key.lower())] is not None +}) + +# Open the template and substitute the placeholders with actual values +with open('Justfile.tmpl', 'r') as f: + template = DblDollars(f.read()) + result = template.substitute(substitutions) + +# Write the result to the output file +with open(file_out, 'w') as f: + f.write(result) + diff --git a/examples/pldb/blocks.feather b/examples/pldb/blocks.feather new file mode 100644 index 00000000..56c2157f Binary files /dev/null and b/examples/pldb/blocks.feather differ diff --git a/go.mod b/go.mod index 87c419b5..5aea3f74 100644 --- a/go.mod +++ b/go.mod @@ -68,6 +68,7 @@ require ( github.com/stretchr/objx v0.5.0 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasttemplate v1.2.1 // indirect + github.com/yuin/goldmark v1.5.4 // indirect go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect go.uber.org/zap v1.17.0 // indirect