tExpand tests to include reorgs and many many txes - electrum-personal-server - Maximally lightweight electrum server for a single user
HTML git clone https://git.parazyd.org/electrum-personal-server
DIR Log
DIR Files
DIR Refs
DIR README
---
DIR commit 5b71d929988cf99a6ff2d5595218e4419c9cc620
DIR parent a057f49693a05187b0a898cc348176513ffcee37
HTML Author: chris-belcher <chris-belcher@users.noreply.github.com>
Date: Tue, 26 Jun 2018 23:31:44 +0100
Expand tests to include reorgs and many many txes
Several new tests are added to test the reorganization-checking code.
Also added is a test which simulates building a history with 1100
ttransactions, and where 130 transactions arrive afterwards.
Some more debug print statements are added where they are useful.
Diffstat:
M electrumpersonalserver/transaction… | 30 +++++++++++++++---------------
M test/test_parse_mpks.py | 6 +++++-
M test/test_transactionmonitor.py | 218 ++++++++++++++++++++++++++-----
3 files changed, 204 insertions(+), 50 deletions(-)
---
DIR diff --git a/electrumpersonalserver/transactionmonitor.py b/electrumpersonalserver/transactionmonitor.py
t@@ -1,6 +1,7 @@
import time, pprint, math, sys
from decimal import Decimal
+from collections import defaultdict
from electrumpersonalserver.jsonrpc import JsonRpcError
import electrumpersonalserver.hashes as hashes
t@@ -155,14 +156,11 @@ class TransactionMonitor(object):
new_history_element["height"], sh_to_add))
count += 1
- unconfirmed_txes = {}
+ unconfirmed_txes = defaultdict(list)
for scrhash, his in address_history.items():
uctx = self.sort_address_history_list(his)
for u in uctx:
- if u["tx_hash"] in unconfirmed_txes:
- unconfirmed_txes[u["tx_hash"]].append(scrhash)
- else:
- unconfirmed_txes[u["tx_hash"]] = [scrhash]
+ unconfirmed_txes[u["tx_hash"]].append(scrhash)
self.debug("unconfirmed_txes = " + str(unconfirmed_txes))
self.debug("reorganizable_txes = " + str(self.reorganizable_txes))
if len(ret) > 0:
t@@ -292,10 +290,7 @@ class TransactionMonitor(object):
#transaction became unconfirmed in a reorg
self.log("A transaction was reorg'd out: " + txid)
elements_removed.append(reorgable_tx)
- if txid in self.unconfirmed_txes:
- self.unconfirmed_txes[txid].extend(scrhashes)
- else:
- self.unconfirmed_txes[txid] = list(scrhashes)
+ self.unconfirmed_txes[txid].extend(scrhashes)
#add to history as unconfirmed
txd = self.rpc.call("decoderawtransaction", [tx["hex"]])
t@@ -312,11 +307,13 @@ class TransactionMonitor(object):
elif tx["blockhash"] != blockhash:
block = self.rpc.call("getblockheader", [tx["blockhash"]])
if block["height"] == height: #reorg but height is the same
+ self.log("A transaction was reorg'd but still confirmed " +
+ "at same height: " + txid)
continue
#reorged but still confirmed at a different height
updated_scrhashes.update(scrhashes)
- self.log("A transaction was reorg'd but still confirmed at " +
- "same height: " + txid)
+ self.log("A transaction was reorg'd but still confirmed to " +
+ "a new block and different height: " + txid)
#update history with the new height
for scrhash in scrhashes:
for h in self.address_history[scrhash]["history"]:
t@@ -383,6 +380,12 @@ class TransactionMonitor(object):
for i in range(max_attempts):
self.debug("listtransactions tx_request_count="
+ str(tx_request_count))
+ ##how listtransactions works
+ ##skip and count parameters take most-recent txes first
+ ## so skip=0 count=1 will return the most recent tx
+ ##and skip=0 count=3 will return the 3 most recent txes
+ ##but the actual list returned has the REVERSED order
+ ##skip=0 count=3 will return a list with the most recent tx LAST
ret = self.rpc.call("listtransactions", ["*", tx_request_count, 0,
True])
ret = ret[::-1]
t@@ -459,10 +462,7 @@ class TransactionMonitor(object):
self.address_history[scrhash]["history"].append(
new_history_element)
if new_history_element["height"] == 0:
- if tx["txid"] in self.unconfirmed_txes:
- self.unconfirmed_txes[tx["txid"]].append(scrhash)
- else:
- self.unconfirmed_txes[tx["txid"]] = [scrhash]
+ self.unconfirmed_txes[tx["txid"]].append(scrhash)
if tx["confirmations"] > 0:
self.reorganizable_txes.append((tx["txid"], tx["blockhash"],
new_history_element["height"], matching_scripthashes))
DIR diff --git a/test/test_parse_mpks.py b/test/test_parse_mpks.py
t@@ -16,7 +16,11 @@ from electrumpersonalserver import parse_electrum_master_public_key
"2 tpubD6NzVbkrYhZ4YVMVzC7wZeRfz3bhqcHvV8M3UiULCfzFtLtp5nwvi6LnBQegrkx" +
"YGPkSzXUEvcPEHcKdda8W1YShVBkhFBGkLxjSQ1Nx3cJ Vpub5fAqpSRkLmvXwqbuR61M" +
"aKMSwj5z5xUBwanaz3qnJ5MgaBDpFSLUvKTiNK9zHpdvrg2LHHXkKxSXBHNWNpZz9b1Vq" +
- "ADjmcCs3arSoxN3F3r" #inconsistent magic
+ "ADjmcCs3arSoxN3F3r", #inconsistent magic
+ "e9d4b7866dd1e91c862aebf62a49548c7dbf7bcc6e4b7b8c9da820c7737968df9c09d" +
+ "5a3e271dc814a29981f81b3faaf2737b551ef5dcc6189cf0f8252c442", #wrong length
+ "e9d4b7866dd1e91c862aebf62a49548c7dbf7bcc6e4b7b8c9da820c7737968df9c09d" +
+ "5a3e271dc814a29981f81b3faaf2737b551ef5dcc6189cf0f8252c442ZZ" #not hex
]
)
DIR diff --git a/test/test_transactionmonitor.py b/test/test_transactionmonitor.py
t@@ -5,6 +5,11 @@ from electrumpersonalserver import (DeterministicWallet, TransactionMonitor,
JsonRpcError, script_to_scripthash)
class DummyJsonRpc(object):
+ """
+ Electrum Personal Server gets all its information about the bitcoin network
+ from the json-rpc interface. This dummy interface is used for simulating
+ events in bitcoin
+ """
def __init__(self, txlist, utxoset, block_heights):
self.txlist = txlist
self.utxoset = utxoset
t@@ -15,7 +20,7 @@ class DummyJsonRpc(object):
if method == "listtransactions":
count = int(params[1])
skip = int(params[2])
- return self.txlist[skip:skip + count]
+ return self.txlist[skip:skip + count][::-1]
elif method == "gettransaction":
for t in self.txlist:
if t["txid"] == params[0]:
t@@ -25,16 +30,19 @@ class DummyJsonRpc(object):
for t in self.txlist:
if t["hex"] == params[0]:
return t
+ debugf(params[0])
assert 0
elif method == "gettxout":
for u in self.utxoset:
if u["txid"] == params[0] and u["vout"] == params[1]:
return u
+ debugf("txid = " + params[0] + " vout = " + str(params[1]))
assert 0
elif method == "getblockheader":
- if params[0] not in self.block_heights:
- assert 0
- return {"height": self.block_heights[params[0]]}
+ if params[0] in self.block_heights:
+ return {"height": self.block_heights[params[0]]}
+ debugf(params[0])
+ assert 0
elif method == "decodescript":
return {"addresses": [dummy_spk_to_address(params[0])]}
elif method == "importaddress":
t@@ -43,7 +51,7 @@ class DummyJsonRpc(object):
raise ValueError("unknown method in dummy jsonrpc")
def add_transaction(self, tx):
- self.txlist.append(tx)
+ self.txlist = [tx] + self.txlist
def get_imported_addresses(self):
return self.imported_addresses
t@@ -62,6 +70,7 @@ class DummyDeterministicWallet(DeterministicWallet):
def dummy_spk_to_address(spk):
+ ##spk is short for scriptPubKey
return spk + "-address"
debugf = lambda x: print("[DEBUG] " + x)
t@@ -97,6 +106,7 @@ def create_dummy_funding_tx(confirmations=1, output_spk=None,
"blockhash": dummy_containing_block,
"hex": "placeholder-test-txhex" + str(dummy_id)
}
+ debugf("created dummy tx: " + str(dummy_tx))
return dummy_spk, containing_block_height, dummy_tx
def assert_address_history_tx(address_history, spk, height, txid, subscribed):
t@@ -139,6 +149,42 @@ def test_two_txes():
height=containing_block_height2, txid=dummy_tx2["txid"],
subscribed=False)
+def test_many_txes():
+ ##many txes in wallet and many more added,, intended to test the loop
+ ## in build_addr_history and check_for_new_txes()
+ input_spk, input_block_height1, input_tx = create_dummy_funding_tx()
+ dummy_spk, containing_block_height, dummy_tx = create_dummy_funding_tx(
+ confirmations=0, input_txid=input_tx["vin"][0])
+ sh = script_to_scripthash(dummy_spk)
+
+ #batch size is 1000
+ INITIAL_TX_COUNT = 1100
+ txes = [dummy_tx]
+ #0confirm to avoid having to obtain block hash
+ txes.extend( (create_dummy_funding_tx(output_spk=dummy_spk,
+ input_txid=input_tx["vin"][0], confirmations=0)[2]
+ for i in range(INITIAL_TX_COUNT-1)) )
+ assert len(txes) == INITIAL_TX_COUNT
+
+ rpc = DummyJsonRpc(txes, [dummy_tx["vin"][0]], {})
+ txmonitor = TransactionMonitor(rpc, deterministic_wallets, debugf, logf)
+ assert txmonitor.build_address_history([dummy_spk])
+ assert len(txmonitor.address_history) == 1
+ assert len(list(txmonitor.check_for_updated_txes())) == 0
+ assert len(txmonitor.address_history[sh]["history"]) == INITIAL_TX_COUNT
+
+ ADDED_TX_COUNT = 130
+ new_txes = []
+ new_txes.extend( (create_dummy_funding_tx(output_spk=dummy_spk,
+ input_txid=input_tx["vin"][0], confirmations=0)[2]
+ for i in range(ADDED_TX_COUNT)) )
+
+ for tx in new_txes:
+ rpc.add_transaction(tx)
+ assert len(list(txmonitor.check_for_updated_txes())) == 0
+ assert len(txmonitor.address_history[sh]["history"]) == (INITIAL_TX_COUNT
+ + ADDED_TX_COUNT)
+
def test_non_subscribed_confirmation():
###one unconfirmed tx in wallet belonging to us, with confirmed inputs,
### addr history built, then tx confirms, not subscribed to address
t@@ -199,6 +245,28 @@ def test_unrelated_tx():
assert len(txmonitor.get_electrum_history(script_to_scripthash(
our_dummy_spk))) == 0
+def test_duplicate_txid():
+ ###two txes with the same txid, built history
+ dummy_spk, containing_block_height1, dummy_tx1 = create_dummy_funding_tx()
+ dummy_spk, containing_block_height2, dummy_tx2 = create_dummy_funding_tx(
+ output_spk=dummy_spk)
+ dummy_spk, containing_block_height3, dummy_tx3 = create_dummy_funding_tx(
+ output_spk=dummy_spk)
+ dummy_tx2["txid"] = dummy_tx1["txid"]
+ dummy_tx3["txid"] = dummy_tx1["txid"]
+ sh = script_to_scripthash(dummy_spk)
+ rpc = DummyJsonRpc([dummy_tx1, dummy_tx2], [], {dummy_tx1["blockhash"]:
+ containing_block_height1, dummy_tx2["blockhash"]: containing_block_height2, dummy_tx3["blockhash"]: containing_block_height3})
+ txmonitor = TransactionMonitor(rpc, deterministic_wallets, debugf, logf)
+ assert txmonitor.build_address_history([dummy_spk])
+ assert len(txmonitor.get_electrum_history(sh)) == 1
+ txmonitor.subscribe_address(sh)
+ assert txmonitor.get_electrum_history(sh)[0]["tx_hash"] == dummy_tx1["txid"]
+ rpc.add_transaction(dummy_tx3)
+ assert len(list(txmonitor.check_for_updated_txes())) == 1
+ assert len(txmonitor.get_electrum_history(sh)) == 1
+ assert txmonitor.get_electrum_history(sh)[0]["tx_hash"] == dummy_tx1["txid"]
+
def test_address_reuse():
###transaction which arrives to an address which already has a tx on it
dummy_spk1, containing_block_height1, dummy_tx1 = create_dummy_funding_tx()
t@@ -284,63 +352,145 @@ def test_conflicted_tx():
###conflicted transaction should get rejected
dummy_spk, containing_block_height, dummy_tx = create_dummy_funding_tx(
confirmations=-1)
-
rpc = DummyJsonRpc([dummy_tx], [], {})
txmonitor = TransactionMonitor(rpc, deterministic_wallets, debugf, logf)
+ sh = script_to_scripthash(dummy_spk)
+
assert txmonitor.build_address_history([dummy_spk])
assert len(txmonitor.address_history) == 1
- assert len(txmonitor.get_electrum_history(script_to_scripthash(
- dummy_spk))) == 0 #shouldnt show up after build history b/c conflicted
+ #shouldnt show up after build history because conflicted
+ assert len(txmonitor.get_electrum_history(sh)) == 0
+
+ dummy_spk, containing_block_height, dummy_tx = create_dummy_funding_tx(
+ confirmations=-1, output_spk=dummy_spk)
rpc.add_transaction(dummy_tx)
assert len(list(txmonitor.check_for_updated_txes())) == 0
- assert len(txmonitor.get_electrum_history(script_to_scripthash(
- dummy_spk))) == 0 #incoming tx is not added too
+ #incoming tx is not added either
+ assert len(txmonitor.get_electrum_history(sh)) == 0
-def test_double_spend():
+def test_reorg_finney_attack():
###an unconfirmed tx being broadcast, another conflicting tx being
### confirmed, the first tx gets conflicted status
- dummy_spk, containing_block_height1, dummy_tx1 = create_dummy_funding_tx(
+ dummy_spk1, containing_block_height1, dummy_tx1 = create_dummy_funding_tx(
confirmations=0)
- dummy_spk_, containing_block_height2, dummy_tx2 = create_dummy_funding_tx(
- confirmations=0, input_txid=dummy_tx1["vin"][0], output_spk=dummy_spk)
+ dummy_spk2, containing_block_height2, dummy_tx2 = create_dummy_funding_tx(
+ confirmations=0, input_txid=dummy_tx1["vin"][0])
#two unconfirmed txes spending the same input, so they are in conflict
rpc = DummyJsonRpc([dummy_tx1], [dummy_tx1["vin"][0]],
{dummy_tx1["blockhash"]: containing_block_height1,
dummy_tx2["blockhash"]: containing_block_height2})
txmonitor = TransactionMonitor(rpc, deterministic_wallets, debugf, logf)
- assert txmonitor.build_address_history([dummy_spk])
- assert len(txmonitor.address_history) == 1
- sh = script_to_scripthash(dummy_spk)
- assert len(txmonitor.get_electrum_history(sh)) == 1
- assert_address_history_tx(txmonitor.address_history, spk=dummy_spk,
+ assert txmonitor.build_address_history([dummy_spk1, dummy_spk2])
+ assert len(txmonitor.address_history) == 2
+ sh1 = script_to_scripthash(dummy_spk1)
+ sh2 = script_to_scripthash(dummy_spk2)
+ assert len(txmonitor.get_electrum_history(sh1)) == 1
+ assert len(txmonitor.get_electrum_history(sh2)) == 0
+ assert_address_history_tx(txmonitor.address_history, spk=dummy_spk1,
height=0, txid=dummy_tx1["txid"], subscribed=False)
# a conflicting transaction confirms
rpc.add_transaction(dummy_tx2)
dummy_tx1["confirmations"] = -1
dummy_tx2["confirmations"] = 1
assert len(list(txmonitor.check_for_updated_txes())) == 0
- assert len(txmonitor.get_electrum_history(sh)) == 1
- assert_address_history_tx(txmonitor.address_history, spk=dummy_spk,
+ assert len(txmonitor.get_electrum_history(sh1)) == 0
+ assert len(txmonitor.get_electrum_history(sh2)) == 1
+ assert_address_history_tx(txmonitor.address_history, spk=dummy_spk2,
+ height=containing_block_height2, txid=dummy_tx2["txid"],
+ subscribed=False)
+
+def test_reorg_race_attack():
+ #a tx is confirmed, a chain reorganization happens and that tx is replaced
+ # by another tx spending the same input, the original tx is now conflicted
+ dummy_spk1, containing_block_height1, dummy_tx1 = create_dummy_funding_tx()
+ dummy_spk2, containing_block_height2, dummy_tx2 = create_dummy_funding_tx(
+ input_txid=dummy_tx1["vin"][0])
+
+ rpc = DummyJsonRpc([dummy_tx1], [],
+ {dummy_tx1["blockhash"]: containing_block_height1,
+ dummy_tx2["blockhash"]: containing_block_height2})
+ txmonitor = TransactionMonitor(rpc, deterministic_wallets, debugf, logf)
+ assert txmonitor.build_address_history([dummy_spk1, dummy_spk2])
+ assert len(txmonitor.address_history) == 2
+ sh1 = script_to_scripthash(dummy_spk1)
+ sh2 = script_to_scripthash(dummy_spk2)
+ assert len(txmonitor.get_electrum_history(sh1)) == 1
+ assert len(txmonitor.get_electrum_history(sh2)) == 0
+ assert_address_history_tx(txmonitor.address_history, spk=dummy_spk1,
+ height=containing_block_height1, txid=dummy_tx1["txid"], subscribed=False)
+ #race attack happens
+ #dummy_tx1 goes to -1 confirmations, dummy_tx2 gets confirmed
+ rpc.add_transaction(dummy_tx2)
+ dummy_tx1["confirmations"] = -1
+ dummy_tx2["confirmations"] = 1
+ assert len(list(txmonitor.check_for_updated_txes())) == 0
+ assert len(txmonitor.get_electrum_history(sh1)) == 0
+ assert len(txmonitor.get_electrum_history(sh2)) == 1
+ assert_address_history_tx(txmonitor.address_history, spk=dummy_spk2,
height=containing_block_height2, txid=dummy_tx2["txid"],
subscribed=False)
+def test_reorg_censor_tx():
+ #confirmed tx gets reorgd out and becomes unconfirmed
+ dummy_spk1, containing_block_height1, dummy_tx1 = create_dummy_funding_tx()
+
+ rpc = DummyJsonRpc([dummy_tx1], [dummy_tx1["vin"][0]],
+ {dummy_tx1["blockhash"]: containing_block_height1})
+ txmonitor = TransactionMonitor(rpc, deterministic_wallets, debugf, logf)
+ assert txmonitor.build_address_history([dummy_spk1])
+ assert len(txmonitor.address_history) == 1
+ sh = script_to_scripthash(dummy_spk1)
+ assert len(txmonitor.get_electrum_history(sh)) == 1
+ assert_address_history_tx(txmonitor.address_history, spk=dummy_spk1,
+ height=containing_block_height1, txid=dummy_tx1["txid"], subscribed=False)
+ #blocks appear which reorg out the tx, making it unconfirmed
+ dummy_tx1["confirmations"] = 0
+ assert len(list(txmonitor.check_for_updated_txes())) == 0
+ assert len(txmonitor.get_electrum_history(sh)) == 1
+ assert_address_history_tx(txmonitor.address_history, spk=dummy_spk1,
+ height=0, txid=dummy_tx1["txid"], subscribed=False)
+
+def test_reorg_different_block():
+ #confirmed tx gets reorged into another block with a different height
+ dummy_spk1, containing_block_height1, dummy_tx1 = create_dummy_funding_tx()
+ dummy_spk2, containing_block_height2, dummy_tx2 = create_dummy_funding_tx()
+
+ rpc = DummyJsonRpc([dummy_tx1], [],
+ {dummy_tx1["blockhash"]: containing_block_height1,
+ dummy_tx2["blockhash"]: containing_block_height2})
+ txmonitor = TransactionMonitor(rpc, deterministic_wallets, debugf, logf)
+ assert txmonitor.build_address_history([dummy_spk1])
+ assert len(txmonitor.address_history) == 1
+ sh = script_to_scripthash(dummy_spk1)
+ assert len(txmonitor.get_electrum_history(sh)) == 1
+ assert_address_history_tx(txmonitor.address_history, spk=dummy_spk1,
+ height=containing_block_height1, txid=dummy_tx1["txid"], subscribed=False)
+
+ #tx gets reorged into another block (so still confirmed)
+ dummy_tx1["blockhash"] = dummy_tx2["blockhash"]
+ assert len(list(txmonitor.check_for_updated_txes())) == 0
+ assert len(txmonitor.get_electrum_history(sh)) == 1
+ assert_address_history_tx(txmonitor.address_history, spk=dummy_spk1,
+ height=containing_block_height2, txid=dummy_tx1["txid"],
+ subscribed=False)
+
+def test_tx_safe_from_reorg():
+ ##tx confirmed with 1 confirmation, then confirmations goes to 100
+ ## test that the reorganizable_txes list length goes down
+ dummy_spk1, containing_block_height1, dummy_tx1 = create_dummy_funding_tx()
+ rpc = DummyJsonRpc([dummy_tx1], [],
+ {dummy_tx1["blockhash"]: containing_block_height1})
+ txmonitor = TransactionMonitor(rpc, deterministic_wallets, debugf, logf)
+ assert txmonitor.build_address_history([dummy_spk1])
+ assert len(list(txmonitor.check_for_updated_txes())) == 0
+ assert len(txmonitor.reorganizable_txes) == 1
+ dummy_tx1["confirmations"] = 2000
+ assert len(list(txmonitor.check_for_updated_txes())) == 0
+ assert len(txmonitor.reorganizable_txes) == 0
#other possible stuff to test:
#finding confirmed and unconfirmed tx, in that order, then both confirm
#finding unconfirmed and confirmed tx, in that order, then both confirm
-#tests about conflicts:
-#build address history where reorgable txes are found
-#an unconfirmed tx arrives, gets confirmed, reaches the safe threshold
-# and gets removed from list
-#a confirmed tx arrives, reaches safe threshold and gets removed
-#an unconfirmed tx arrives, confirms, gets reorgd out, returns to
-# unconfirmed
-#an unconfirmed tx arrives, confirms, gets reorgd out and conflicted
-#an unconfirmed tx arrives, confirms, gets reorgd out and confirmed at
-# a different height
-#an unconfirmed tx arrives, confirms, gets reorgd out and confirmed in
-# the same height
-