tMerge pull request #4405 from SomberNight/remove_from_addresses_from_wallet2 - electrum - Electrum Bitcoin wallet HTML git clone https://git.parazyd.org/electrum DIR Log DIR Files DIR Refs DIR Submodules --- DIR commit ebdce9fb9a9fddb18ca0f235b44b8b359180a5ae DIR parent 79558c11706f4a6d2eec1fecf746454cd44b25b0 HTML Author: ThomasV <thomasv@electrum.org> Date: Mon, 11 Jun 2018 19:33:27 +0200 Merge pull request #4405 from SomberNight/remove_from_addresses_from_wallet2 remove "from addresses" Diffstat: M gui/qt/transaction_dialog.py | 13 ++++--------- M lib/storage.py | 34 +++++++++++++++++++++++++++---- M lib/tests/test_transaction.py | 18 +++++++++--------- M lib/tests/test_wallet_vertical.py | 40 +++++++++++++------------------- M lib/transaction.py | 116 ++++++++++++++++++------------- M lib/util.py | 2 ++ M lib/wallet.py | 200 +++++++++++++++---------------- M plugins/trustedcoin/trustedcoin.py | 3 +-- 8 files changed, 224 insertions(+), 202 deletions(-) --- DIR diff --git a/gui/qt/transaction_dialog.py b/gui/qt/transaction_dialog.py t@@ -83,7 +83,7 @@ class TxDialog(QDialog, MessageBoxMixin): self.saved = False self.desc = desc - self.setMinimumWidth(750) + self.setMinimumWidth(950) self.setWindowTitle(_("Transaction")) vbox = QVBoxLayout() t@@ -293,15 +293,10 @@ class TxDialog(QDialog, MessageBoxMixin): else: prevout_hash = x.get('prevout_hash') prevout_n = x.get('prevout_n') - cursor.insertText(prevout_hash[0:8] + '...', ext) - cursor.insertText(prevout_hash[-8:] + ":%-4d " % prevout_n, ext) - addr = x.get('address') - if addr == "(pubkey)": - _addr = self.wallet.get_txin_address(x) - if _addr: - addr = _addr + cursor.insertText(prevout_hash + ":%-4d " % prevout_n, ext) + addr = self.wallet.get_txin_address(x) if addr is None: - addr = _('unknown') + addr = '' cursor.insertText(addr, text_format(addr)) if x.get('value'): cursor.insertText(format_amount(x['value']), ext) DIR diff --git a/lib/storage.py b/lib/storage.py t@@ -32,7 +32,9 @@ import stat import pbkdf2, hmac, hashlib import base64 import zlib +from collections import defaultdict +from . import util from .util import PrintError, profiler, InvalidPassword, WalletFileException, bfh from .plugins import run_hook, plugin_loaders from .keystore import bip44_derivation t@@ -44,7 +46,7 @@ from . import ecc OLD_SEED_VERSION = 4 # electrum versions < 2.0 NEW_SEED_VERSION = 11 # electrum versions >= 2.0 -FINAL_SEED_VERSION = 16 # electrum >= 2.7 will set this to prevent +FINAL_SEED_VERSION = 17 # electrum >= 2.7 will set this to prevent # old versions from overwriting new format t@@ -225,8 +227,8 @@ class WalletStorage(PrintError): def put(self, key, value): try: - json.dumps(key) - json.dumps(value) + json.dumps(key, cls=util.MyEncoder) + json.dumps(value, cls=util.MyEncoder) except: self.print_error("json error: cannot save", key) return t@@ -250,7 +252,7 @@ class WalletStorage(PrintError): return if not self.modified: return - s = json.dumps(self.data, indent=4, sort_keys=True) + s = json.dumps(self.data, indent=4, sort_keys=True, cls=util.MyEncoder) if self.pubkey: s = bytes(s, 'utf8') c = zlib.compress(s) t@@ -329,6 +331,7 @@ class WalletStorage(PrintError): def requires_upgrade(self): return self.file_exists() and self.get_seed_version() < FINAL_SEED_VERSION + @profiler def upgrade(self): self.print_error('upgrading wallet format') t@@ -339,6 +342,7 @@ class WalletStorage(PrintError): self.convert_version_14() self.convert_version_15() self.convert_version_16() + self.convert_version_17() self.put('seed_version', FINAL_SEED_VERSION) # just to be sure self.write() t@@ -531,6 +535,28 @@ class WalletStorage(PrintError): self.put('seed_version', 16) + def convert_version_17(self): + # delete pruned_txo; construct spent_outpoints + if not self._is_upgrade_method_needed(16, 16): + return + + self.put('pruned_txo', None) + + from .transaction import Transaction + transactions = self.get('transactions', {}) # txid -> raw_tx + spent_outpoints = defaultdict(dict) + for txid, raw_tx in transactions.items(): + tx = Transaction(raw_tx) + for txin in tx.inputs(): + if txin['type'] == 'coinbase': + continue + prevout_hash = txin['prevout_hash'] + prevout_n = txin['prevout_n'] + spent_outpoints[prevout_hash][prevout_n] = txid + self.put('spent_outpoints', spent_outpoints) + + self.put('seed_version', 17) + def convert_imported(self): if not self._is_upgrade_method_needed(0, 13): return DIR diff --git a/lib/tests/test_transaction.py b/lib/tests/test_transaction.py t@@ -8,7 +8,7 @@ from lib.util import bh2u, bfh from . import SequentialTestCase from .test_bitcoin import needs_test_with_all_ecc_implementations -unsigned_blob = '01000000012a5c9a94fcde98f5581cd00162c60a13936ceb75389ea65bf38633b424eb4031000000005701ff4c53ff0488b21e03ef2afea18000000089689bff23e1e7fb2f161daa37270a97a3d8c2e537584b2d304ecb47b86d21fc021b010d3bd425f8cf2e04824bfdf1f1f5ff1d51fadd9a41f9e3fb8dd3403b1bfe00000000ffffffff0140420f00000000001976a914230ac37834073a42146f11ef8414ae929feaafc388ac00000000' +unsigned_blob = '45505446ff0001000000012a5c9a94fcde98f5581cd00162c60a13936ceb75389ea65bf38633b424eb4031000000005701ff4c53ff0488b21e03ef2afea18000000089689bff23e1e7fb2f161daa37270a97a3d8c2e537584b2d304ecb47b86d21fc021b010d3bd425f8cf2e04824bfdf1f1f5ff1d51fadd9a41f9e3fb8dd3403b1bfe00000000ffffffff0140420f00000000001976a914230ac37834073a42146f11ef8414ae929feaafc388ac00000000' signed_blob = '01000000012a5c9a94fcde98f5581cd00162c60a13936ceb75389ea65bf38633b424eb4031000000006c493046022100a82bbc57a0136751e5433f41cf000b3f1a99c6744775e76ec764fb78c54ee100022100f9e80b7de89de861dc6fb0c1429d5da72c2b6b2ee2406bc9bfb1beedd729d985012102e61d176da16edd1d258a200ad9759ef63adf8e14cd97f53227bae35cdb84d2f6ffffffff0140420f00000000001976a914230ac37834073a42146f11ef8414ae929feaafc388ac00000000' v2_blob = "0200000001191601a44a81e061502b7bfbc6eaa1cef6d1e6af5308ef96c9342f71dbf4b9b5000000006b483045022100a6d44d0a651790a477e75334adfb8aae94d6612d01187b2c02526e340a7fd6c8022028bdf7a64a54906b13b145cd5dab21a26bd4b85d6044e9b97bceab5be44c2a9201210253e8e0254b0c95776786e40984c1aa32a7d03efa6bdacdea5f421b774917d346feffffff026b20fa04000000001976a914024db2e87dd7cfd0e5f266c5f212e21a31d805a588aca0860100000000001976a91421919b94ae5cefcdf0271191459157cdb41c4cbf88aca6240700" signed_segwit_blob = "01000000000101b66d722484f2db63e827ebf41d02684fed0c6550e85015a6c9d41ef216a8a6f00000000000fdffffff0280c3c90100000000160014b65ce60857f7e7892b983851c2a8e3526d09e4ab64bac30400000000160014c478ebbc0ab2097706a98e10db7cf101839931c4024730440220789c7d47f876638c58d98733c30ae9821c8fa82b470285dcdf6db5994210bf9f02204163418bbc44af701212ad42d884cc613f3d3d831d2d0cc886f767cca6e0235e012103083a6dc250816d771faa60737bfe78b23ad619f6b458e0a1f1688e3a0605e79c00000000" t@@ -78,7 +78,9 @@ class TestTransaction(SequentialTestCase): 'scriptPubKey': '76a914230ac37834073a42146f11ef8414ae929feaafc388ac', 'type': TYPE_ADDRESS, 'value': 1000000}], - 'version': 1 + 'partial': True, + 'segwit_ser': False, + 'version': 1, } tx = transaction.Transaction(unsigned_blob) self.assertEqual(tx.deserialize(), expected) t@@ -105,17 +107,13 @@ class TestTransaction(SequentialTestCase): @needs_test_with_all_ecc_implementations def test_tx_signed(self): expected = { - 'inputs': [{ - 'type': 'p2pkh', - 'address': '1446oU3z268EeFgfcwJv6X2VBXHfoYxfuD', - 'num_sig': 1, + 'inputs': [{'address': None, + 'num_sig': 0, 'prevout_hash': '3140eb24b43386f35ba69e3875eb6c93130ac66201d01c58f598defc949a5c2a', 'prevout_n': 0, - 'pubkeys': ['02e61d176da16edd1d258a200ad9759ef63adf8e14cd97f53227bae35cdb84d2f6'], 'scriptSig': '493046022100a82bbc57a0136751e5433f41cf000b3f1a99c6744775e76ec764fb78c54ee100022100f9e80b7de89de861dc6fb0c1429d5da72c2b6b2ee2406bc9bfb1beedd729d985012102e61d176da16edd1d258a200ad9759ef63adf8e14cd97f53227bae35cdb84d2f6', 'sequence': 4294967295, - 'signatures': ['3046022100a82bbc57a0136751e5433f41cf000b3f1a99c6744775e76ec764fb78c54ee100022100f9e80b7de89de861dc6fb0c1429d5da72c2b6b2ee2406bc9bfb1beedd729d98501'], - 'x_pubkeys': ['02e61d176da16edd1d258a200ad9759ef63adf8e14cd97f53227bae35cdb84d2f6']}], + 'type': 'unknown'}], 'lockTime': 0, 'outputs': [{ 'address': '14CHYaaByjJZpx4oHBpfDMdqhTyXnZ3kVs', t@@ -123,6 +121,8 @@ class TestTransaction(SequentialTestCase): 'scriptPubKey': '76a914230ac37834073a42146f11ef8414ae929feaafc388ac', 'type': TYPE_ADDRESS, 'value': 1000000}], + 'partial': False, + 'segwit_ser': False, 'version': 1 } tx = transaction.Transaction(signed_blob) DIR diff --git a/lib/tests/test_wallet_vertical.py b/lib/tests/test_wallet_vertical.py t@@ -486,10 +486,9 @@ class TestWalletSending(TestCaseForTestnet): self.assertTrue(tx.is_complete()) self.assertTrue(tx.is_segwit()) self.assertEqual(1, len(tx.inputs())) + self.assertEqual(wallet1.txin_type, tx.inputs()[0]['type']) tx_copy = Transaction(tx.serialize()) - self.assertEqual(wallet1.is_mine(tx.inputs()[0]['address']), wallet1.is_mine(tx_copy.inputs()[0]['address'])) - self.assertTrue(wallet1.is_mine(tx.inputs()[0]['address'])) - self.assertEqual(wallet1.txin_type, tx_copy.inputs()[0]['type']) + self.assertTrue(wallet1.is_mine(wallet1.get_txin_address(tx_copy.inputs()[0]))) self.assertEqual('010000000001010392c1940e2ec9f2372919ca3887327fe5b98b866022cc79bab5cbed5a53d2ad0000000000feffffff0290d00300000000001976a914ea7804a2c266063572cc009a63dc25dcc0e9d9b588ac285e0b0000000000160014690b59a8140602fb23cc2904ece9cc4daf361052024730440220608a5339ca894592da82119e1e4a1d09335d70a552c683687223b8ed724465e902201b3f0feccf391b1b6257e4b18970ae57d7ca060af2dae519b3690baad2b2a34e0121030faee9b4a25b7db82023ca989192712cdd4cb53d3d9338591c7909e581ae1c0c00000000', str(tx_copy)) t@@ -507,10 +506,9 @@ class TestWalletSending(TestCaseForTestnet): self.assertTrue(tx.is_complete()) self.assertFalse(tx.is_segwit()) self.assertEqual(1, len(tx.inputs())) + self.assertEqual(wallet2.txin_type, tx.inputs()[0]['type']) tx_copy = Transaction(tx.serialize()) - self.assertEqual(wallet2.is_mine(tx.inputs()[0]['address']), wallet2.is_mine(tx_copy.inputs()[0]['address'])) - self.assertTrue(wallet2.is_mine(tx.inputs()[0]['address'])) - self.assertEqual(wallet2.txin_type, tx_copy.inputs()[0]['type']) + self.assertTrue(wallet2.is_mine(wallet2.get_txin_address(tx_copy.inputs()[0]))) self.assertEqual('0100000001e228327e4c0bb80661d258d625f516307e7c127c7f3e2b476a22e89b4dae063c000000006b483045022100d3895b31e7c9766987c6f53794c7394f534f4acecefda5479d963236f9703d0b022026dd4e40700ceb788f136faf54bf85b966648dc7c2a608d8110604f2d22d59070121030b482838721a38d94847699fed8818b5c5f56500ef72f13489e365b65e5749cffeffffff02a0860100000000001600148a28bddb7f61864bdcf58b2ad13d5aeb3abc3c4268360200000000001976a914ca4c60999c46c2108326590b125aefd476dcb11888ac00000000', str(tx_copy)) t@@ -564,10 +562,9 @@ class TestWalletSending(TestCaseForTestnet): self.assertTrue(tx.is_complete()) self.assertFalse(tx.is_segwit()) self.assertEqual(1, len(tx.inputs())) + self.assertEqual(wallet1a.txin_type, tx.inputs()[0]['type']) tx_copy = Transaction(tx.serialize()) - self.assertEqual(wallet1a.is_mine(tx.inputs()[0]['address']), wallet1a.is_mine(tx_copy.inputs()[0]['address'])) - self.assertTrue(wallet1a.is_mine(tx.inputs()[0]['address'])) - self.assertEqual(wallet1a.txin_type, tx_copy.inputs()[0]['type']) + self.assertTrue(wallet1a.is_mine(wallet1a.get_txin_address(tx_copy.inputs()[0]))) self.assertEqual('01000000017120d4e1f2cdfe7df000d632cff74167fb354f0546d5cfc228e5c98756d55cb201000000fdfe0000483045022100f9ce5616683e613ae14b98d56436454b003348a8172e2ed598018e3d206e57d7022030c65c6551e839f9e9409812be624dbb4e36bd4152c9ed9b0988c10fd8201d1401483045022100d5cb94d4d1dcf01bb9e9280e8178a7e9ada3ad14378ca543afcc9f5667b27cb2022018e76b74800a21934e73b226b34cbbe45c877fba64693da8a20d3cb330f2eafd014c69522102afb4af9a91264e1c6dce3ebe5312801723270ac0ba8134b7b49129328fcb0f2821030b482838721a38d94847699fed8818b5c5f56500ef72f13489e365b65e5749cf2103e5db7969ae2f2576e6a061bf3bb2db16571e77ffb41e0b27170734359235cbce53aefeffffff0250a50500000000001976a9149cd3dfb0d87a861770ae4e268e74b45335cf00ab88ac2862b1000000000017a9142e517854aa54668128c0e9a3fdd4dec13ad571368700000000', str(tx_copy)) t@@ -585,10 +582,9 @@ class TestWalletSending(TestCaseForTestnet): self.assertTrue(tx.is_complete()) self.assertFalse(tx.is_segwit()) self.assertEqual(1, len(tx.inputs())) + self.assertEqual(wallet2.txin_type, tx.inputs()[0]['type']) tx_copy = Transaction(tx.serialize()) - self.assertEqual(wallet2.is_mine(tx.inputs()[0]['address']), wallet2.is_mine(tx_copy.inputs()[0]['address'])) - self.assertTrue(wallet2.is_mine(tx.inputs()[0]['address'])) - self.assertEqual(wallet2.txin_type, tx_copy.inputs()[0]['type']) + self.assertTrue(wallet2.is_mine(wallet2.get_txin_address(tx_copy.inputs()[0]))) self.assertEqual('01000000015df26ee0f55487ca29727c50dbf0ce2227d3e3eb44621219ff1c2e40d0bdf326000000008b483045022100bd9f61ba82507d3a28922fb8be129e14699dfa54ddd03cc9494f696d38ac4121022071afca6fad5bc5c09b0a675e6444be3e97dbbdbc283764ee5f4e27a032d933d80141045f7ba332df2a7b4f5d13f246e307c9174cfa9b8b05f3b83410a3c23ef8958d610be285963d67c7bc1feb082f168fa9877c25999963ff8b56b242a852b23e25edfeffffff02a08601000000000017a91480c2353f6a7bc3c71e99e062655b19adb3dd2e4887280b0400000000001976a914ca14915184a2662b5d1505ce7142c8ca066c70e288ac00000000', str(tx_copy)) t@@ -657,10 +653,9 @@ class TestWalletSending(TestCaseForTestnet): self.assertTrue(tx.is_complete()) self.assertTrue(tx.is_segwit()) self.assertEqual(1, len(tx.inputs())) + self.assertEqual(wallet1a.txin_type, tx.inputs()[0]['type']) tx_copy = Transaction(tx.serialize()) - self.assertEqual(wallet1a.is_mine(tx.inputs()[0]['address']), wallet1a.is_mine(tx_copy.inputs()[0]['address'])) - self.assertTrue(wallet1a.is_mine(tx.inputs()[0]['address'])) - self.assertEqual(wallet1a.txin_type, tx_copy.inputs()[0]['type']) + self.assertTrue(wallet1a.is_mine(wallet1a.get_txin_address(tx_copy.inputs()[0]))) self.assertEqual('01000000000101213e1012a461e056752fab5a6414a2fb63f950cd21a50ac5e2b82d339d6cbdd20000000000feffffff023075000000000000220020cc5e4cc05a76d0648cd0742768556317e9f8cc729aed077134287909035dba88888402000000000017a914187842cea9c15989a51ce7ca889a08b824bf8743870400483045022100ea2fbd3d8681cfafdcae1bdaaa64f92fb9872fb8f6bf03a2b7effcf7390b66c8022021a79eff7975479934f958f3766d6ac61d708c79b785e398b3bcd84b1039e9b501483045022100dbc4f1ec18f0e0deb4ff88d7d5b3d3b7b500a80d0c0f33efbd3262f0c8689095022074fd226c0b52e3716ad907d14cba9c79aca482a8f4a51662ca83a5b9db49e15b016952210223f815ab09f6bfc8519165c5232947ae89d9d43d678fb3486f3b28382a2371fa210273c529c2c9a99592f2066cebc2172a48991af2b471cb726b9df78c6497ce984e2102aa8fc578b445a1e4257be6b978fcece92980def98dce0e1eb89e7364635ae94153ae00000000', str(tx_copy)) t@@ -681,10 +676,9 @@ class TestWalletSending(TestCaseForTestnet): self.assertTrue(tx.is_complete()) self.assertTrue(tx.is_segwit()) self.assertEqual(1, len(tx.inputs())) + self.assertEqual(wallet2a.txin_type, tx.inputs()[0]['type']) tx_copy = Transaction(tx.serialize()) - self.assertEqual(wallet2a.is_mine(tx.inputs()[0]['address']), wallet2a.is_mine(tx_copy.inputs()[0]['address'])) - self.assertTrue(wallet2a.is_mine(tx.inputs()[0]['address'])) - self.assertEqual(wallet2a.txin_type, tx_copy.inputs()[0]['type']) + self.assertTrue(wallet2a.is_mine(wallet2a.get_txin_address(tx_copy.inputs()[0]))) self.assertEqual('0100000000010149d077be0ee9d52776211e9b4fec1cc02bd53661a04e120a97db8b78d83c9c6e01000000232200204311edae835c7a5aa712c8ca644180f13a3b2f3b420fa879b181474724d6163cfeffffff0260ea00000000000017a9143025051b6b5ccd4baf30dfe2de8aa84f0dd567ed87a0860100000000002200203c43ac80d6e3015cf378bf6bac0c22456723d6050bef324ec641e7762440c63c0400483045022100c254468bbe6b8bd1c8c01b6a223e46cc5c6b56fbba87d59575385ad249133b0e02207139688f8d6ae8076c92a266d98454d25c040d04c8e513a37bf7c32dad3e48210147304402204af5edbab2d674f6a9edef8c97b2f7fdf8ababedc7b287710cc7a64d4699358b022064e2d07f4bb32373be31b2003dc56b7b831a7c01419326efb3011c64b898b3f00147522102119f899075a131d4d519d4cdcf5de5907dc2df3b93d54b53ded852211d2b6cb12102fdb0f6775d4b6619257c43343ba5e7807b0164f1eb3f00f2b594ab9e53ab812652ae00000000', str(tx_copy)) t@@ -730,10 +724,9 @@ class TestWalletSending(TestCaseForTestnet): self.assertTrue(tx.is_complete()) self.assertFalse(tx.is_segwit()) self.assertEqual(1, len(tx.inputs())) + self.assertEqual(wallet1a.txin_type, tx.inputs()[0]['type']) tx_copy = Transaction(tx.serialize()) - self.assertEqual(wallet1a.is_mine(tx.inputs()[0]['address']), wallet1a.is_mine(tx_copy.inputs()[0]['address'])) - self.assertTrue(wallet1a.is_mine(tx.inputs()[0]['address'])) - self.assertEqual(wallet1a.txin_type, tx_copy.inputs()[0]['type']) + self.assertTrue(wallet1a.is_mine(wallet1a.get_txin_address(tx_copy.inputs()[0]))) self.assertEqual('0100000001a391c8b3d4a551eac85714f3f0a7514381c014ba4688de085b0fcee42dc13711010000009200483045022100fcf03aeb97b66791372c18aa0dd651817cf458d941dd628c966f0305a023360f022016c534530e267b6a52f90e62aa9fb50ace609ffb21e472d3ba7b29db9b30050e014751210245c90e040d4f9d1fc136b3d4d6b7535bbb5df2bd27666c21977042cc1e05b5b02103c9a6bebfce6294488315e58137a279b2efe09f1f528ecf93b40675ded3cf0e5f52aefeffffff0240420f000000000017a9149573eb50f3136dff141ac304190f41c8becc92ce8738b32d000000000017a914b815d1b430ae9b632e3834ed537f7956325ee2a98700000000', str(tx_copy)) t@@ -751,10 +744,9 @@ class TestWalletSending(TestCaseForTestnet): self.assertTrue(tx.is_complete()) self.assertTrue(tx.is_segwit()) self.assertEqual(1, len(tx.inputs())) + self.assertEqual(wallet2.txin_type, tx.inputs()[0]['type']) tx_copy = Transaction(tx.serialize()) - self.assertEqual(wallet2.is_mine(tx.inputs()[0]['address']), wallet2.is_mine(tx_copy.inputs()[0]['address'])) - self.assertTrue(wallet2.is_mine(tx.inputs()[0]['address'])) - self.assertEqual(wallet2.txin_type, tx_copy.inputs()[0]['type']) + self.assertTrue(wallet2.is_mine(wallet2.get_txin_address(tx_copy.inputs()[0]))) self.assertEqual('010000000001012a4a7e0487c839f211a2f174aa91e94146bdfd408d9271e3d481960b86947e1b00000000171600149fad840ed174584ee054bd26f3e411817338c5edfeffffff02e09304000000000017a914b0b9f31bace76cdfae2c14abc03e223403d7dc4b87d89a0a000000000017a9148ccd0efb2be5b412c4033715f560ed8f446c8ceb87024830450221009c816c3e0c40b37085244f0976f65635b8d711952bad9843c5f51e386fd37cc402202c34a4a7227182742d9f93e9f28c4bd30ded6514550f39614cb5ad00e46690070121038362bbf0b4918b37e9d7c75930ed3a78e3d445724cb5c37ade4a59b6e411fe4e00000000', str(tx_copy)) DIR diff --git a/lib/transaction.py b/lib/transaction.py t@@ -44,6 +44,7 @@ import sys from .keystore import xpubkey_to_address, xpubkey_to_pubkey NO_SIGNATURE = 'ff' +PARTIAL_TXN_HEADER_MAGIC = b'EPTF\xff' class SerializationError(Exception): t@@ -383,15 +384,6 @@ def parse_scriptSig(d, _bytes): bh2u(_bytes)) -def _revise_txin_type_guess_for_txin(txin): - _type = txin.get('type', 'unknown') - # fix incorrect guess of p2sh-segwit - we_guessed_segwit_input_type = Transaction.is_segwit_inputtype(_type) - has_zero_witness = txin.get('witness', '00') in ('00', None) - if we_guessed_segwit_input_type and has_zero_witness: - txin['type'] = 'unknown' - - def parse_redeemScript_multisig(redeem_script: bytes): dec2 = [ x for x in script_GetOp(redeem_script) ] try: t@@ -443,7 +435,7 @@ def get_address_from_output_script(_bytes, *, net=None): return TYPE_SCRIPT, bh2u(_bytes) -def parse_input(vds): +def parse_input(vds, full_parse: bool): d = {} prevout_hash = hash_encode(vds.read_bytes(32)) prevout_n = vds.read_uint32() t@@ -451,23 +443,22 @@ def parse_input(vds): sequence = vds.read_uint32() d['prevout_hash'] = prevout_hash d['prevout_n'] = prevout_n + d['scriptSig'] = bh2u(scriptSig) d['sequence'] = sequence + d['type'] = 'unknown' if prevout_hash != '00'*32 else 'coinbase' + d['address'] = None + d['num_sig'] = 0 + if not full_parse: + return d d['x_pubkeys'] = [] d['pubkeys'] = [] d['signatures'] = {} - d['address'] = None - d['num_sig'] = 0 - d['scriptSig'] = bh2u(scriptSig) - if prevout_hash == '00'*32: - d['type'] = 'coinbase' - else: - d['type'] = 'unknown' - if scriptSig: - try: - parse_scriptSig(d, scriptSig) - except BaseException: - traceback.print_exc(file=sys.stderr) - print_error('failed to parse scriptSig', bh2u(scriptSig)) + if d['type'] != 'coinbase' and scriptSig: + try: + parse_scriptSig(d, scriptSig) + except BaseException: + traceback.print_exc(file=sys.stderr) + print_error('failed to parse scriptSig', bh2u(scriptSig)) return d t@@ -483,28 +474,24 @@ def construct_witness(items: Sequence[Union[str, int, bytes]]) -> str: return witness -def parse_witness(vds, txin): +def parse_witness(vds, txin, full_parse: bool): n = vds.read_compact_size() if n == 0: + txin['witness'] = '00' return if n == 0xffffffff: txin['value'] = vds.read_uint64() + txin['witness_version'] = vds.read_uint16() n = vds.read_compact_size() # now 'n' is the number of items in the witness w = list(bh2u(vds.read_bytes(vds.read_compact_size())) for i in range(n)) - txin['witness'] = construct_witness(w) + if not full_parse: + return - # FIXME: witness version > 0 will probably fail here. - # For native segwit, we would need the scriptPubKey of the parent txn - # to determine witness program version, and properly parse the witness. - # In case of p2sh-segwit, we can tell based on the scriptSig in this txn. - # The code below assumes witness version 0. - # p2sh-segwit should work in that case; for native segwit we need to tell - # between p2wpkh and p2wsh; we do this based on number of witness items, - # hence (FIXME) p2wsh with n==2 (maybe n==1 ?) will probably fail. - # If v==0 and n==2, we need parent scriptPubKey to distinguish between p2wpkh and p2wsh. try: + if txin['witness_version'] != 0: + raise UnknownTxinType() if txin['type'] == 'coinbase': pass elif txin['type'] == 'p2wsh-p2sh' or n > 2: t@@ -533,7 +520,6 @@ def parse_witness(vds, txin): raise UnknownTxinType() except UnknownTxinType: txin['type'] = 'unknown' - # FIXME: GUI might show 'unknown' address (e.g. for a non-multisig p2wsh) except BaseException: txin['type'] = 'unknown' traceback.print_exc(file=sys.stderr) t@@ -550,11 +536,21 @@ def parse_output(vds, i): return d -def deserialize(raw): - vds = BCDataStream() - vds.write(bfh(raw)) +def deserialize(raw: str, force_full_parse=False) -> dict: + raw_bytes = bfh(raw) d = {} - start = vds.read_cursor + if raw_bytes[:5] == PARTIAL_TXN_HEADER_MAGIC: + d['partial'] = is_partial = True + partial_format_version = raw_bytes[5] + if partial_format_version != 0: + raise SerializationError('unknown tx partial serialization format version: {}' + .format(partial_format_version)) + raw_bytes = raw_bytes[6:] + else: + d['partial'] = is_partial = False + full_parse = force_full_parse or is_partial + vds = BCDataStream() + vds.write(raw_bytes) d['version'] = vds.read_int32() n_vin = vds.read_compact_size() is_segwit = (n_vin == 0) t@@ -563,17 +559,15 @@ def deserialize(raw): if marker != b'\x01': raise ValueError('invalid txn marker byte: {}'.format(marker)) n_vin = vds.read_compact_size() - d['inputs'] = [parse_input(vds) for i in range(n_vin)] + d['segwit_ser'] = is_segwit + d['inputs'] = [parse_input(vds, full_parse=full_parse) for i in range(n_vin)] n_vout = vds.read_compact_size() d['outputs'] = [parse_output(vds, i) for i in range(n_vout)] if is_segwit: for i in range(n_vin): txin = d['inputs'][i] - parse_witness(vds, txin) + parse_witness(vds, txin, full_parse=full_parse) d['lockTime'] = vds.read_uint32() - for i in range(n_vin): - txin = d['inputs'][i] - _revise_txin_type_guess_for_txin(txin) return d t@@ -613,6 +607,10 @@ class Transaction: self._outputs = None self.locktime = 0 self.version = 1 + # by default we assume this is a partial txn; + # this value will get properly set when deserializing + self.is_partial_originally = True + self._segwit_ser = None # None means "don't know" def update(self, raw): self.raw = raw t@@ -645,7 +643,9 @@ class Transaction: def update_signatures(self, raw): """Add new signatures to a transaction""" - d = deserialize(raw) + if self.is_complete(): + return + d = deserialize(raw, force_full_parse=True) for i, txin in enumerate(self.inputs()): pubkeys, x_pubkeys = self.get_sorted_pubkeys(txin) sigs1 = txin.get('signatures') t@@ -689,6 +689,8 @@ class Transaction: self._outputs = [(x['type'], x['address'], x['value']) for x in d['outputs']] self.locktime = d['lockTime'] self.version = d['version'] + self.is_partial_originally = d['partial'] + self._segwit_ser = d['segwit_ser'] return d @classmethod t@@ -779,10 +781,12 @@ class Transaction: else: raise Exception('wrong txin type:', txin['type']) if self.is_txin_complete(txin) or estimate_size: - value_field = '' + partial_format_witness_prefix = '' else: - value_field = var_int(0xffffffff) + int_to_hex(txin['value'], 8) - return value_field + witness + input_value = int_to_hex(txin['value'], 8) + witness_version = int_to_hex(txin.get('witness_version', 0), 2) + partial_format_witness_prefix = var_int(0xffffffff) + input_value + witness_version + return partial_format_witness_prefix + witness @classmethod def is_segwit_input(cls, txin): t@@ -842,6 +846,8 @@ class Transaction: if txin['type'] == 'coinbase': return True num_sig = txin.get('num_sig', 1) + if num_sig == 0: + return True x_signatures = txin['signatures'] signatures = list(filter(None, x_signatures)) return len(signatures) == num_sig t@@ -932,9 +938,21 @@ class Transaction: return preimage def is_segwit(self): + if not self.is_partial_originally: + return self._segwit_ser return any(self.is_segwit_input(x) for x in self.inputs()) def serialize(self, estimate_size=False, witness=True): + network_ser = self.serialize_to_network(estimate_size, witness) + if estimate_size: + return network_ser + if self.is_partial_originally and not self.is_complete(): + partial_format_version = '00' + return bh2u(PARTIAL_TXN_HEADER_MAGIC) + partial_format_version + network_ser + else: + return network_ser + + def serialize_to_network(self, estimate_size=False, witness=True): nVersion = int_to_hex(self.version, 4) nLocktime = int_to_hex(self.locktime, 4) inputs = self.inputs() t@@ -1056,6 +1074,8 @@ class Transaction: return s, r def is_complete(self): + if not self.is_partial_originally: + return True s, r = self.signature_count() return r == s DIR diff --git a/lib/util.py b/lib/util.py t@@ -158,6 +158,8 @@ class MyEncoder(json.JSONEncoder): return str(obj) if isinstance(obj, datetime): return obj.isoformat(' ')[:-3] + if isinstance(obj, set): + return list(obj) return super(MyEncoder, self).default(obj) class PrintError(object): DIR diff --git a/lib/wallet.py b/lib/wallet.py t@@ -106,7 +106,7 @@ def append_utxos_to_inputs(inputs, network, pubkey, txin_type, imax): item['address'] = address item['type'] = txin_type item['prevout_hash'] = item['tx_hash'] - item['prevout_n'] = item['tx_pos'] + item['prevout_n'] = int(item['tx_pos']) item['pubkeys'] = [pubkey] item['x_pubkeys'] = [pubkey] item['signatures'] = [None] t@@ -170,11 +170,6 @@ class UnrelatedTransactionException(AddTransactionException): return _("Transaction is unrelated to this wallet.") -class NotIsMineTransactionException(AddTransactionException): - def __str__(self): - return _("Only transactions with inputs owned by the wallet can be added.") - - class Abstract_Wallet(PrintError): """ Wallet classes are created to handle various address generation methods. t@@ -216,7 +211,6 @@ class Abstract_Wallet(PrintError): self.test_addresses_sanity() self.load_transactions() self.load_local_history() - self.build_spent_outpoints() self.check_history() self.load_unverified_transactions() self.remove_local_transactions_we_dont_have() t@@ -249,19 +243,29 @@ class Abstract_Wallet(PrintError): @profiler def load_transactions(self): + # load txi, txo, tx_fees self.txi = self.storage.get('txi', {}) + for txid, d in list(self.txi.items()): + for addr, lst in d.items(): + self.txi[txid][addr] = set([tuple(x) for x in lst]) self.txo = self.storage.get('txo', {}) self.tx_fees = self.storage.get('tx_fees', {}) - self.pruned_txo = self.storage.get('pruned_txo', {}) tx_list = self.storage.get('transactions', {}) + # load transactions self.transactions = {} for tx_hash, raw in tx_list.items(): tx = Transaction(raw) self.transactions[tx_hash] = tx - if self.txi.get(tx_hash) is None and self.txo.get(tx_hash) is None \ - and (tx_hash not in self.pruned_txo.values()): + if self.txi.get(tx_hash) is None and self.txo.get(tx_hash) is None: self.print_error("removing unreferenced tx", tx_hash) self.transactions.pop(tx_hash) + # load spent_outpoints + _spent_outpoints = self.storage.get('spent_outpoints', {}) + self.spent_outpoints = defaultdict(dict) + for prevout_hash, d in _spent_outpoints.items(): + for prevout_n_str, spending_txid in d.items(): + prevout_n = int(prevout_n_str) + self.spent_outpoints[prevout_hash][prevout_n] = spending_txid @profiler def load_local_history(self): t@@ -286,8 +290,8 @@ class Abstract_Wallet(PrintError): self.storage.put('txi', self.txi) self.storage.put('txo', self.txo) self.storage.put('tx_fees', self.tx_fees) - self.storage.put('pruned_txo', self.pruned_txo) self.storage.put('addr_history', self.history) + self.storage.put('spent_outpoints', self.spent_outpoints) if write: self.storage.write() t@@ -303,22 +307,13 @@ class Abstract_Wallet(PrintError): self.txi = {} self.txo = {} self.tx_fees = {} - self.pruned_txo = {} - self.spent_outpoints = {} + self.spent_outpoints = defaultdict(dict) self.history = {} self.verified_tx = {} self.transactions = {} self.save_transactions() @profiler - def build_spent_outpoints(self): - self.spent_outpoints = {} - for txid, items in self.txi.items(): - for addr, l in items.items(): - for ser, v in l: - self.spent_outpoints[ser] = txid - - @profiler def check_history(self): save = False t@@ -333,11 +328,11 @@ class Abstract_Wallet(PrintError): hist = self.history[addr] for tx_hash, tx_height in hist: - if tx_hash in self.pruned_txo.values() or self.txi.get(tx_hash) or self.txo.get(tx_hash): + if self.txi.get(tx_hash) or self.txo.get(tx_hash): continue tx = self.transactions.get(tx_hash) if tx is not None: - self.add_transaction(tx_hash, tx) + self.add_transaction(tx_hash, tx, allow_unrelated=True) save = True if save: self.save_transactions() t@@ -528,9 +523,6 @@ class Abstract_Wallet(PrintError): def get_tx_delta(self, tx_hash, address): "effect of tx on address" - # pruned - if tx_hash in self.pruned_txo.values(): - return None delta = 0 # substract the value of coins sent from address d = self.txi.get(tx_hash, {}).get(address, []) t@@ -561,7 +553,7 @@ class Abstract_Wallet(PrintError): is_partial = False v_in = v_out = v_out_mine = 0 for txin in tx.inputs(): - addr = txin.get('address') + addr = self.get_txin_address(txin) if self.is_mine(addr): is_mine = True is_relevant = True t@@ -786,7 +778,7 @@ class Abstract_Wallet(PrintError): def get_txin_address(self, txi): addr = txi.get('address') - if addr != "(pubkey)": + if addr and addr != "(pubkey)": return addr prevout_hash = txi.get('prevout_hash') prevout_n = txi.get('prevout_n') t@@ -794,8 +786,8 @@ class Abstract_Wallet(PrintError): for addr, l in dd.items(): for n, v, is_cb in l: if n == prevout_n: - self.print_error("found pay-to-pubkey address:", addr) return addr + return None def get_txout_address(self, txo): _type, x, v = txo t@@ -815,14 +807,15 @@ class Abstract_Wallet(PrintError): """ conflicting_txns = set() with self.transaction_lock: - for txi in tx.inputs(): - ser = Transaction.get_outpoint_from_txin(txi) - if ser is None: + for txin in tx.inputs(): + if txin['type'] == 'coinbase': continue - spending_tx_hash = self.spent_outpoints.get(ser, None) + prevout_hash = txin['prevout_hash'] + prevout_n = txin['prevout_n'] + spending_tx_hash = self.spent_outpoints[prevout_hash].get(prevout_n) if spending_tx_hash is None: continue - # this outpoint (ser) has already been spent, by spending_tx + # this outpoint has already been spent, by spending_tx assert spending_tx_hash in self.transactions conflicting_txns |= {spending_tx_hash} txid = tx.txid() t@@ -833,7 +826,7 @@ class Abstract_Wallet(PrintError): conflicting_txns -= {txid} return conflicting_txns - def add_transaction(self, tx_hash, tx): + def add_transaction(self, tx_hash, tx, allow_unrelated=False): assert tx_hash, tx_hash assert tx, tx assert tx.is_complete() t@@ -846,15 +839,14 @@ class Abstract_Wallet(PrintError): # being is_mine, as we roll the gap_limit forward is_coinbase = tx.inputs()[0]['type'] == 'coinbase' tx_height = self.get_tx_height(tx_hash)[0] - is_mine = any([self.is_mine(txin['address']) for txin in tx.inputs()]) - # do not save if tx is local and not mine - if tx_height == TX_HEIGHT_LOCAL and not is_mine: - # FIXME the test here should be for "not all is_mine"; cannot detect conflict in some cases - raise NotIsMineTransactionException() - # raise exception if unrelated to wallet - is_for_me = any([self.is_mine(self.get_txout_address(txo)) for txo in tx.outputs()]) - if not is_mine and not is_for_me: - raise UnrelatedTransactionException() + if not allow_unrelated: + # note that during sync, if the transactions are not properly sorted, + # it could happen that we think tx is unrelated but actually one of the inputs is is_mine. + # this is the main motivation for allow_unrelated + is_mine = any([self.is_mine(self.get_txin_address(txin)) for txin in tx.inputs()]) + is_for_me = any([self.is_mine(self.get_txout_address(txo)) for txo in tx.outputs()]) + if not is_mine and not is_for_me: + raise UnrelatedTransactionException() # Find all conflicting transactions. # In case of a conflict, # 1. confirmed > mempool > local t@@ -884,26 +876,27 @@ class Abstract_Wallet(PrintError): for tx_hash2 in to_remove: self.remove_transaction(tx_hash2) # add inputs + def add_value_from_prev_output(): + dd = self.txo.get(prevout_hash, {}) + # note: this nested loop takes linear time in num is_mine outputs of prev_tx + for addr, outputs in dd.items(): + # note: instead of [(n, v, is_cb), ...]; we could store: {n -> (v, is_cb)} + for n, v, is_cb in outputs: + if n == prevout_n: + if addr and self.is_mine(addr): + if d.get(addr) is None: + d[addr] = set() + d[addr].add((ser, v)) + return self.txi[tx_hash] = d = {} for txi in tx.inputs(): - addr = self.get_txin_address(txi) - if txi['type'] != 'coinbase': - prevout_hash = txi['prevout_hash'] - prevout_n = txi['prevout_n'] - ser = prevout_hash + ':%d'%prevout_n - if addr and self.is_mine(addr): - # we only track is_mine spends - self.spent_outpoints[ser] = tx_hash - # find value from prev output - dd = self.txo.get(prevout_hash, {}) - for n, v, is_cb in dd.get(addr, []): - if n == prevout_n: - if d.get(addr) is None: - d[addr] = [] - d[addr].append((ser, v)) - break - else: - self.pruned_txo[ser] = tx_hash + if txi['type'] == 'coinbase': + continue + prevout_hash = txi['prevout_hash'] + prevout_n = txi['prevout_n'] + ser = prevout_hash + ':%d' % prevout_n + self.spent_outpoints[prevout_hash][prevout_n] = tx_hash + add_value_from_prev_output() # add outputs self.txo[tx_hash] = d = {} for n, txo in enumerate(tx.outputs()): t@@ -914,15 +907,15 @@ class Abstract_Wallet(PrintError): if d.get(addr) is None: d[addr] = [] d[addr].append((n, v, is_coinbase)) - # give v to txi that spends me - next_tx = self.pruned_txo.get(ser) - if next_tx is not None: - self.pruned_txo.pop(ser) - dd = self.txi.get(next_tx, {}) - if dd.get(addr) is None: - dd[addr] = [] - dd[addr].append((ser, v)) - self._add_tx_to_local_history(next_tx) + # give v to txi that spends me + next_tx = self.spent_outpoints[tx_hash].get(n) + if next_tx is not None: + dd = self.txi.get(next_tx, {}) + if dd.get(addr) is None: + dd[addr] = set() + if (ser, v) not in dd[addr]: + dd[addr].add((ser, v)) + self._add_tx_to_local_history(next_tx) # add to local history self._add_tx_to_local_history(tx_hash) # save t@@ -930,43 +923,41 @@ class Abstract_Wallet(PrintError): return True def remove_transaction(self, tx_hash): + def remove_from_spent_outpoints(): + # undo spends in spent_outpoints + if tx is not None: # if we have the tx, this branch is faster + for txin in tx.inputs(): + if txin['type'] == 'coinbase': + continue + prevout_hash = txin['prevout_hash'] + prevout_n = txin['prevout_n'] + self.spent_outpoints[prevout_hash].pop(prevout_n, None) + if not self.spent_outpoints[prevout_hash]: + self.spent_outpoints.pop(prevout_hash) + else: # expensive but always works + for prevout_hash, d in list(self.spent_outpoints.items()): + for prevout_n, spending_txid in d.items(): + if spending_txid == tx_hash: + self.spent_outpoints[prevout_hash].pop(prevout_n, None) + if not self.spent_outpoints[prevout_hash]: + self.spent_outpoints.pop(prevout_hash) + # Remove this tx itself; if nothing spends from it. + # It is not so clear what to do if other txns spend from it, but it will be + # removed when those other txns are removed. + if not self.spent_outpoints[tx_hash]: + self.spent_outpoints.pop(tx_hash) with self.transaction_lock: self.print_error("removing tx from history", tx_hash) - self.transactions.pop(tx_hash, None) - # undo spent_outpoints that are in txi - for addr, l in self.txi[tx_hash].items(): - for ser, v in l: - self.spent_outpoints.pop(ser, None) - # undo spent_outpoints that are in pruned_txo - for ser, hh in list(self.pruned_txo.items()): - if hh == tx_hash: - self.spent_outpoints.pop(ser, None) - self.pruned_txo.pop(ser) - + tx = self.transactions.pop(tx_hash, None) + remove_from_spent_outpoints() self._remove_tx_from_local_history(tx_hash) - - # add tx to pruned_txo, and undo the txi addition - for next_tx, dd in self.txi.items(): - for addr, l in list(dd.items()): - ll = l[:] - for item in ll: - ser, v = item - prev_hash, prev_n = ser.split(':') - if prev_hash == tx_hash: - l.remove(item) - self.pruned_txo[ser] = next_tx - if l == []: - dd.pop(addr) - else: - dd[addr] = l - self.txi.pop(tx_hash, None) self.txo.pop(tx_hash, None) def receive_tx_callback(self, tx_hash, tx, tx_height): self.add_unverified_tx(tx_hash, tx_height) - self.add_transaction(tx_hash, tx) + self.add_transaction(tx_hash, tx, allow_unrelated=True) def receive_history_callback(self, addr, hist, tx_fees): with self.lock: t@@ -978,10 +969,6 @@ class Abstract_Wallet(PrintError): self.verified_tx.pop(tx_hash, None) if self.verifier: self.verifier.remove_spv_proof_for_tx(tx_hash) - # but remove completely if not is_mine - if self.txi[tx_hash] == {}: - # FIXME the test here should be for "not all is_mine"; cannot detect conflict in some cases - self.remove_transaction(tx_hash) self.history[addr] = hist for tx_hash, tx_height in hist: t@@ -989,8 +976,9 @@ class Abstract_Wallet(PrintError): self.add_unverified_tx(tx_hash, tx_height) # if addr is new, we have to recompute txi and txo tx = self.transactions.get(tx_hash) - if tx is not None and self.txi.get(tx_hash, {}).get(addr) is None and self.txo.get(tx_hash, {}).get(addr) is None: - self.add_transaction(tx_hash, tx) + if tx is None: + continue + self.add_transaction(tx_hash, tx, allow_unrelated=True) # Store fees self.tx_fees.update(tx_fees) t@@ -1787,6 +1775,7 @@ class Abstract_Wallet(PrintError): def get_depending_transactions(self, tx_hash): """Returns all (grand-)children of tx_hash in this wallet.""" children = set() + # TODO rewrite this to use self.spent_outpoints for other_hash, tx in self.transactions.items(): for input in (tx.inputs()): if input["prevout_hash"] == tx_hash: t@@ -1975,7 +1964,6 @@ class Imported_Wallet(Simple_Wallet): self.verified_tx.pop(tx_hash, None) self.unverified_tx.pop(tx_hash, None) self.transactions.pop(tx_hash, None) - # FIXME: what about pruned_txo? self.storage.put('verified_tx3', self.verified_tx) self.save_transactions() DIR diff --git a/plugins/trustedcoin/trustedcoin.py b/plugins/trustedcoin/trustedcoin.py t@@ -286,8 +286,7 @@ class Wallet_2fa(Multisig_Wallet): return otp = int(otp) long_user_id, short_id = self.get_user_id() - tx_dict = tx.as_dict() - raw_tx = tx_dict["hex"] + raw_tx = tx.serialize_to_network() r = server.sign(short_id, raw_tx, otp) if r: raw_tx = r.get('transaction')