twallet.py - electrum - Electrum Bitcoin wallet
HTML git clone https://git.parazyd.org/electrum
DIR Log
DIR Files
DIR Refs
DIR Submodules
---
twallet.py (136312B)
---
1 # Electrum - lightweight Bitcoin client
2 # Copyright (C) 2015 Thomas Voegtlin
3 #
4 # Permission is hereby granted, free of charge, to any person
5 # obtaining a copy of this software and associated documentation files
6 # (the "Software"), to deal in the Software without restriction,
7 # including without limitation the rights to use, copy, modify, merge,
8 # publish, distribute, sublicense, and/or sell copies of the Software,
9 # and to permit persons to whom the Software is furnished to do so,
10 # subject to the following conditions:
11 #
12 # The above copyright notice and this permission notice shall be
13 # included in all copies or substantial portions of the Software.
14 #
15 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
16 # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
17 # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
18 # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
19 # BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
20 # ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
21 # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 # SOFTWARE.
23
24 # Wallet classes:
25 # - Imported_Wallet: imported addresses or single keys, 0 or 1 keystore
26 # - Standard_Wallet: one HD keystore, P2PKH-like scripts
27 # - Multisig_Wallet: several HD keystores, M-of-N OP_CHECKMULTISIG scripts
28
29 import os
30 import sys
31 import random
32 import time
33 import json
34 import copy
35 import errno
36 import traceback
37 import operator
38 import math
39 from functools import partial
40 from collections import defaultdict
41 from numbers import Number
42 from decimal import Decimal
43 from typing import TYPE_CHECKING, List, Optional, Tuple, Union, NamedTuple, Sequence, Dict, Any, Set
44 from abc import ABC, abstractmethod
45 import itertools
46 import threading
47 import enum
48
49 from aiorpcx import TaskGroup, timeout_after, TaskTimeout, ignore_after
50
51 from .i18n import _
52 from .bip32 import BIP32Node, convert_bip32_intpath_to_strpath, convert_bip32_path_to_list_of_uint32
53 from .crypto import sha256
54 from . import util
55 from .util import (NotEnoughFunds, UserCancelled, profiler,
56 format_satoshis, format_fee_satoshis, NoDynamicFeeEstimates,
57 WalletFileException, BitcoinException, MultipleSpendMaxTxOutputs,
58 InvalidPassword, format_time, timestamp_to_datetime, Satoshis,
59 Fiat, bfh, bh2u, TxMinedInfo, quantize_feerate, create_bip21_uri, OrderedDictWithIndex)
60 from .util import get_backup_dir
61 from .simple_config import SimpleConfig, FEE_RATIO_HIGH_WARNING, FEERATE_WARNING_HIGH_FEE
62 from .bitcoin import COIN, TYPE_ADDRESS
63 from .bitcoin import is_address, address_to_script, is_minikey, relayfee, dust_threshold
64 from .crypto import sha256d
65 from . import keystore
66 from .keystore import (load_keystore, Hardware_KeyStore, KeyStore, KeyStoreWithMPK,
67 AddressIndexGeneric, CannotDerivePubkey)
68 from .util import multisig_type
69 from .storage import StorageEncryptionVersion, WalletStorage
70 from .wallet_db import WalletDB
71 from . import transaction, bitcoin, coinchooser, paymentrequest, ecc, bip32
72 from .transaction import (Transaction, TxInput, UnknownTxinType, TxOutput,
73 PartialTransaction, PartialTxInput, PartialTxOutput, TxOutpoint)
74 from .plugin import run_hook
75 from .address_synchronizer import (AddressSynchronizer, TX_HEIGHT_LOCAL,
76 TX_HEIGHT_UNCONF_PARENT, TX_HEIGHT_UNCONFIRMED, TX_HEIGHT_FUTURE)
77 from .invoices import Invoice, OnchainInvoice, LNInvoice
78 from .invoices import PR_PAID, PR_UNPAID, PR_UNKNOWN, PR_EXPIRED, PR_UNCONFIRMED, PR_TYPE_ONCHAIN, PR_TYPE_LN
79 from .contacts import Contacts
80 from .interface import NetworkException
81 from .mnemonic import Mnemonic
82 from .logging import get_logger
83 from .lnworker import LNWallet
84 from .paymentrequest import PaymentRequest
85 from .util import read_json_file, write_json_file, UserFacingException
86
87 if TYPE_CHECKING:
88 from .network import Network
89 from .exchange_rate import FxThread
90
91
92 _logger = get_logger(__name__)
93
94 TX_STATUS = [
95 _('Unconfirmed'),
96 _('Unconfirmed parent'),
97 _('Not Verified'),
98 _('Local'),
99 ]
100
101
102 class BumpFeeStrategy(enum.Enum):
103 COINCHOOSER = enum.auto()
104 DECREASE_CHANGE = enum.auto()
105 DECREASE_PAYMENT = enum.auto()
106
107
108 async def _append_utxos_to_inputs(*, inputs: List[PartialTxInput], network: 'Network',
109 pubkey: str, txin_type: str, imax: int) -> None:
110 if txin_type in ('p2pkh', 'p2wpkh', 'p2wpkh-p2sh'):
111 address = bitcoin.pubkey_to_address(txin_type, pubkey)
112 scripthash = bitcoin.address_to_scripthash(address)
113 elif txin_type == 'p2pk':
114 script = bitcoin.public_key_to_p2pk_script(pubkey)
115 scripthash = bitcoin.script_to_scripthash(script)
116 else:
117 raise Exception(f'unexpected txin_type to sweep: {txin_type}')
118
119 async def append_single_utxo(item):
120 prev_tx_raw = await network.get_transaction(item['tx_hash'])
121 prev_tx = Transaction(prev_tx_raw)
122 prev_txout = prev_tx.outputs()[item['tx_pos']]
123 if scripthash != bitcoin.script_to_scripthash(prev_txout.scriptpubkey.hex()):
124 raise Exception('scripthash mismatch when sweeping')
125 prevout_str = item['tx_hash'] + ':%d' % item['tx_pos']
126 prevout = TxOutpoint.from_str(prevout_str)
127 txin = PartialTxInput(prevout=prevout)
128 txin.utxo = prev_tx
129 txin.block_height = int(item['height'])
130 txin.script_type = txin_type
131 txin.pubkeys = [bfh(pubkey)]
132 txin.num_sig = 1
133 if txin_type == 'p2wpkh-p2sh':
134 txin.redeem_script = bfh(bitcoin.p2wpkh_nested_script(pubkey))
135 inputs.append(txin)
136
137 u = await network.listunspent_for_scripthash(scripthash)
138 async with TaskGroup() as group:
139 for item in u:
140 if len(inputs) >= imax:
141 break
142 await group.spawn(append_single_utxo(item))
143
144
145 async def sweep_preparations(privkeys, network: 'Network', imax=100):
146
147 async def find_utxos_for_privkey(txin_type, privkey, compressed):
148 pubkey = ecc.ECPrivkey(privkey).get_public_key_hex(compressed=compressed)
149 await _append_utxos_to_inputs(
150 inputs=inputs,
151 network=network,
152 pubkey=pubkey,
153 txin_type=txin_type,
154 imax=imax)
155 keypairs[pubkey] = privkey, compressed
156
157 inputs = [] # type: List[PartialTxInput]
158 keypairs = {}
159 async with TaskGroup() as group:
160 for sec in privkeys:
161 txin_type, privkey, compressed = bitcoin.deserialize_privkey(sec)
162 await group.spawn(find_utxos_for_privkey(txin_type, privkey, compressed))
163 # do other lookups to increase support coverage
164 if is_minikey(sec):
165 # minikeys don't have a compressed byte
166 # we lookup both compressed and uncompressed pubkeys
167 await group.spawn(find_utxos_for_privkey(txin_type, privkey, not compressed))
168 elif txin_type == 'p2pkh':
169 # WIF serialization does not distinguish p2pkh and p2pk
170 # we also search for pay-to-pubkey outputs
171 await group.spawn(find_utxos_for_privkey('p2pk', privkey, compressed))
172 if not inputs:
173 raise UserFacingException(_('No inputs found.'))
174 return inputs, keypairs
175
176
177 async def sweep(
178 privkeys,
179 *,
180 network: 'Network',
181 config: 'SimpleConfig',
182 to_address: str,
183 fee: int = None,
184 imax=100,
185 locktime=None,
186 tx_version=None
187 ) -> PartialTransaction:
188 inputs, keypairs = await sweep_preparations(privkeys, network, imax)
189 total = sum(txin.value_sats() for txin in inputs)
190 if fee is None:
191 outputs = [PartialTxOutput(scriptpubkey=bfh(bitcoin.address_to_script(to_address)),
192 value=total)]
193 tx = PartialTransaction.from_io(inputs, outputs)
194 fee = config.estimate_fee(tx.estimated_size())
195 if total - fee < 0:
196 raise Exception(_('Not enough funds on address.') + '\nTotal: %d satoshis\nFee: %d'%(total, fee))
197 if total - fee < dust_threshold(network):
198 raise Exception(_('Not enough funds on address.') + '\nTotal: %d satoshis\nFee: %d\nDust Threshold: %d'%(total, fee, dust_threshold(network)))
199
200 outputs = [PartialTxOutput(scriptpubkey=bfh(bitcoin.address_to_script(to_address)),
201 value=total - fee)]
202 if locktime is None:
203 locktime = get_locktime_for_new_transaction(network)
204
205 tx = PartialTransaction.from_io(inputs, outputs, locktime=locktime, version=tx_version)
206 rbf = bool(config.get('use_rbf', True))
207 tx.set_rbf(rbf)
208 tx.sign(keypairs)
209 return tx
210
211
212 def get_locktime_for_new_transaction(network: 'Network') -> int:
213 # if no network or not up to date, just set locktime to zero
214 if not network:
215 return 0
216 chain = network.blockchain()
217 if chain.is_tip_stale():
218 return 0
219 # discourage "fee sniping"
220 locktime = chain.height()
221 # sometimes pick locktime a bit further back, to help privacy
222 # of setups that need more time (offline/multisig/coinjoin/...)
223 if random.randint(0, 9) == 0:
224 locktime = max(0, locktime - random.randint(0, 99))
225 return locktime
226
227
228
229 class CannotBumpFee(Exception):
230 def __str__(self):
231 return _('Cannot bump fee') + ':\n\n' + Exception.__str__(self)
232
233 class CannotDoubleSpendTx(Exception):
234 def __str__(self):
235 return _('Cannot cancel transaction') + ':\n\n' + Exception.__str__(self)
236
237 class CannotCPFP(Exception):
238 def __str__(self):
239 return _('Cannot create child transaction') + ':\n\n' + Exception.__str__(self)
240
241 class InternalAddressCorruption(Exception):
242 def __str__(self):
243 return _("Wallet file corruption detected. "
244 "Please restore your wallet from seed, and compare the addresses in both files")
245
246
247 class TxWalletDetails(NamedTuple):
248 txid: Optional[str]
249 status: str
250 label: str
251 can_broadcast: bool
252 can_bump: bool
253 can_cpfp: bool
254 can_dscancel: bool # whether user can double-spend to self
255 can_save_as_local: bool
256 amount: Optional[int]
257 fee: Optional[int]
258 tx_mined_status: TxMinedInfo
259 mempool_depth_bytes: Optional[int]
260 can_remove: bool # whether user should be allowed to delete tx
261 is_lightning_funding_tx: bool
262
263
264 class Abstract_Wallet(AddressSynchronizer, ABC):
265 """
266 Wallet classes are created to handle various address generation methods.
267 Completion states (watching-only, single account, no seed, etc) are handled inside classes.
268 """
269
270 LOGGING_SHORTCUT = 'w'
271 max_change_outputs = 3
272 gap_limit_for_change = 10
273
274 txin_type: str
275 wallet_type: str
276 lnworker: Optional['LNWallet']
277
278 def __init__(self, db: WalletDB, storage: Optional[WalletStorage], *, config: SimpleConfig):
279 if not db.is_ready_to_be_used_by_wallet():
280 raise Exception("storage not ready to be used by Abstract_Wallet")
281
282 self.config = config
283 assert self.config is not None, "config must not be None"
284 self.db = db
285 self.storage = storage
286 # load addresses needs to be called before constructor for sanity checks
287 db.load_addresses(self.wallet_type)
288 self.keystore = None # type: Optional[KeyStore] # will be set by load_keystore
289 AddressSynchronizer.__init__(self, db)
290
291 # saved fields
292 self.use_change = db.get('use_change', True)
293 self.multiple_change = db.get('multiple_change', False)
294 self._labels = db.get_dict('labels')
295 self._frozen_addresses = set(db.get('frozen_addresses', []))
296 self._frozen_coins = db.get_dict('frozen_coins') # type: Dict[str, bool]
297 self.fiat_value = db.get_dict('fiat_value')
298 self.receive_requests = db.get_dict('payment_requests') # type: Dict[str, Invoice]
299 self.invoices = db.get_dict('invoices') # type: Dict[str, Invoice]
300 self._reserved_addresses = set(db.get('reserved_addresses', []))
301
302 self._freeze_lock = threading.Lock() # for mutating/iterating frozen_{addresses,coins}
303
304 self._prepare_onchain_invoice_paid_detection()
305 self.calc_unused_change_addresses()
306 # save wallet type the first time
307 if self.db.get('wallet_type') is None:
308 self.db.put('wallet_type', self.wallet_type)
309 self.contacts = Contacts(self.db)
310 self._coin_price_cache = {}
311
312 self.lnworker = None
313
314 def save_db(self):
315 if self.storage:
316 self.db.write(self.storage)
317
318 def save_backup(self):
319 backup_dir = get_backup_dir(self.config)
320 if backup_dir is None:
321 return
322 new_db = WalletDB(self.db.dump(), manual_upgrades=False)
323
324 if self.lnworker:
325 channel_backups = new_db.get_dict('channel_backups')
326 for chan_id, chan in self.lnworker.channels.items():
327 channel_backups[chan_id.hex()] = self.lnworker.create_channel_backup(chan_id)
328 new_db.put('channels', None)
329 new_db.put('lightning_privkey2', None)
330
331 new_path = os.path.join(backup_dir, self.basename() + '.backup')
332 new_storage = WalletStorage(new_path)
333 new_storage._encryption_version = self.storage._encryption_version
334 new_storage.pubkey = self.storage.pubkey
335 new_db.set_modified(True)
336 new_db.write(new_storage)
337 return new_path
338
339 def has_lightning(self):
340 return bool(self.lnworker)
341
342 def can_have_lightning(self):
343 # we want static_remotekey to be a wallet address
344 return self.txin_type == 'p2wpkh'
345
346 def init_lightning(self):
347 assert self.can_have_lightning()
348 if self.db.get('lightning_privkey2'):
349 return
350 # TODO derive this deterministically from wallet.keystore at keystore generation time
351 # probably along a hardened path ( lnd-equivalent would be m/1017'/coinType'/ )
352 seed = os.urandom(32)
353 node = BIP32Node.from_rootseed(seed, xtype='standard')
354 ln_xprv = node.to_xprv()
355 self.db.put('lightning_privkey2', ln_xprv)
356
357 async def stop(self):
358 """Stop all networking and save DB to disk."""
359 try:
360 async with ignore_after(5):
361 await super().stop()
362 if self.network:
363 if self.lnworker:
364 await self.lnworker.stop()
365 self.lnworker = None
366 finally: # even if we get cancelled
367 if any([ks.is_requesting_to_be_rewritten_to_wallet_file for ks in self.get_keystores()]):
368 self.save_keystore()
369 self.save_db()
370
371 def set_up_to_date(self, b):
372 super().set_up_to_date(b)
373 if b: self.save_db()
374
375 def clear_history(self):
376 super().clear_history()
377 self.save_db()
378
379 def start_network(self, network):
380 AddressSynchronizer.start_network(self, network)
381 if network:
382 if self.lnworker:
383 self.lnworker.start_network(network)
384 # only start gossiping when we already have channels
385 if self.db.get('channels'):
386 self.network.start_gossip()
387
388 def load_and_cleanup(self):
389 self.load_keystore()
390 self.test_addresses_sanity()
391 super().load_and_cleanup()
392
393 @abstractmethod
394 def load_keystore(self) -> None:
395 pass
396
397 def diagnostic_name(self):
398 return self.basename()
399
400 def __str__(self):
401 return self.basename()
402
403 def get_master_public_key(self):
404 return None
405
406 def get_master_public_keys(self):
407 return []
408
409 def basename(self) -> str:
410 return self.storage.basename() if self.storage else 'no name'
411
412 def test_addresses_sanity(self) -> None:
413 addrs = self.get_receiving_addresses()
414 if len(addrs) > 0:
415 addr = str(addrs[0])
416 if not bitcoin.is_address(addr):
417 neutered_addr = addr[:5] + '..' + addr[-2:]
418 raise WalletFileException(f'The addresses in this wallet are not bitcoin addresses.\n'
419 f'e.g. {neutered_addr} (length: {len(addr)})')
420
421 def check_returned_address_for_corruption(func):
422 def wrapper(self, *args, **kwargs):
423 addr = func(self, *args, **kwargs)
424 self.check_address_for_corruption(addr)
425 return addr
426 return wrapper
427
428 def calc_unused_change_addresses(self) -> Sequence[str]:
429 """Returns a list of change addresses to choose from, for usage in e.g. new transactions.
430 The caller should give priority to earlier ones in the list.
431 """
432 with self.lock:
433 # We want a list of unused change addresses.
434 # As a performance optimisation, to avoid checking all addresses every time,
435 # we maintain a list of "not old" addresses ("old" addresses have deeply confirmed history),
436 # and only check those.
437 if not hasattr(self, '_not_old_change_addresses'):
438 self._not_old_change_addresses = self.get_change_addresses()
439 self._not_old_change_addresses = [addr for addr in self._not_old_change_addresses
440 if not self.address_is_old(addr)]
441 unused_addrs = [addr for addr in self._not_old_change_addresses
442 if not self.is_used(addr) and not self.is_address_reserved(addr)]
443 return unused_addrs
444
445 def is_deterministic(self) -> bool:
446 return self.keystore.is_deterministic()
447
448 def _set_label(self, key: str, value: Optional[str]) -> None:
449 with self.lock:
450 if value is None:
451 self._labels.pop(key, None)
452 else:
453 self._labels[key] = value
454
455 def set_label(self, name: str, text: str = None) -> bool:
456 if not name:
457 return False
458 changed = False
459 with self.lock:
460 old_text = self._labels.get(name)
461 if text:
462 text = text.replace("\n", " ")
463 if old_text != text:
464 self._labels[name] = text
465 changed = True
466 else:
467 if old_text is not None:
468 self._labels.pop(name)
469 changed = True
470 if changed:
471 run_hook('set_label', self, name, text)
472 return changed
473
474 def import_labels(self, path):
475 data = read_json_file(path)
476 for key, value in data.items():
477 self.set_label(key, value)
478
479 def export_labels(self, path):
480 write_json_file(path, self.get_all_labels())
481
482 def set_fiat_value(self, txid, ccy, text, fx, value_sat):
483 if not self.db.get_transaction(txid):
484 return
485 # since fx is inserting the thousands separator,
486 # and not util, also have fx remove it
487 text = fx.remove_thousands_separator(text)
488 def_fiat = self.default_fiat_value(txid, fx, value_sat)
489 formatted = fx.ccy_amount_str(def_fiat, commas=False)
490 def_fiat_rounded = Decimal(formatted)
491 reset = not text
492 if not reset:
493 try:
494 text_dec = Decimal(text)
495 text_dec_rounded = Decimal(fx.ccy_amount_str(text_dec, commas=False))
496 reset = text_dec_rounded == def_fiat_rounded
497 except:
498 # garbage. not resetting, but not saving either
499 return False
500 if reset:
501 d = self.fiat_value.get(ccy, {})
502 if d and txid in d:
503 d.pop(txid)
504 else:
505 # avoid saving empty dict
506 return True
507 else:
508 if ccy not in self.fiat_value:
509 self.fiat_value[ccy] = {}
510 self.fiat_value[ccy][txid] = text
511 return reset
512
513 def get_fiat_value(self, txid, ccy):
514 fiat_value = self.fiat_value.get(ccy, {}).get(txid)
515 try:
516 return Decimal(fiat_value)
517 except:
518 return
519
520 def is_mine(self, address) -> bool:
521 if not address: return False
522 return bool(self.get_address_index(address))
523
524 def is_change(self, address) -> bool:
525 if not self.is_mine(address):
526 return False
527 return self.get_address_index(address)[0] == 1
528
529 @abstractmethod
530 def get_address_index(self, address: str) -> Optional[AddressIndexGeneric]:
531 pass
532
533 @abstractmethod
534 def get_address_path_str(self, address: str) -> Optional[str]:
535 """Returns derivation path str such as "m/0/5" to address,
536 or None if not applicable.
537 """
538 pass
539
540 @abstractmethod
541 def get_redeem_script(self, address: str) -> Optional[str]:
542 pass
543
544 @abstractmethod
545 def get_witness_script(self, address: str) -> Optional[str]:
546 pass
547
548 @abstractmethod
549 def get_txin_type(self, address: str) -> str:
550 """Return script type of wallet address."""
551 pass
552
553 def export_private_key(self, address: str, password: Optional[str]) -> str:
554 if self.is_watching_only():
555 raise Exception(_("This is a watching-only wallet"))
556 if not is_address(address):
557 raise Exception(f"Invalid bitcoin address: {address}")
558 if not self.is_mine(address):
559 raise Exception(_('Address not in wallet.') + f' {address}')
560 index = self.get_address_index(address)
561 pk, compressed = self.keystore.get_private_key(index, password)
562 txin_type = self.get_txin_type(address)
563 serialized_privkey = bitcoin.serialize_privkey(pk, compressed, txin_type)
564 return serialized_privkey
565
566 def export_private_key_for_path(self, path: Union[Sequence[int], str], password: Optional[str]) -> str:
567 raise Exception("this wallet is not deterministic")
568
569 @abstractmethod
570 def get_public_keys(self, address: str) -> Sequence[str]:
571 pass
572
573 def get_public_keys_with_deriv_info(self, address: str) -> Dict[bytes, Tuple[KeyStoreWithMPK, Sequence[int]]]:
574 """Returns a map: pubkey -> (keystore, derivation_suffix)"""
575 return {}
576
577 def get_tx_info(self, tx: Transaction) -> TxWalletDetails:
578 tx_wallet_delta = self.get_wallet_delta(tx)
579 is_relevant = tx_wallet_delta.is_relevant
580 is_any_input_ismine = tx_wallet_delta.is_any_input_ismine
581 fee = tx_wallet_delta.fee
582 exp_n = None
583 can_broadcast = False
584 can_bump = False
585 can_cpfp = False
586 tx_hash = tx.txid() # note: txid can be None! e.g. when called from GUI tx dialog
587 is_lightning_funding_tx = False
588 if self.has_lightning() and tx_hash is not None:
589 is_lightning_funding_tx = any([chan.funding_outpoint.txid == tx_hash
590 for chan in self.lnworker.channels.values()])
591 tx_we_already_have_in_db = self.db.get_transaction(tx_hash)
592 can_save_as_local = (is_relevant and tx.txid() is not None
593 and (tx_we_already_have_in_db is None or not tx_we_already_have_in_db.is_complete()))
594 label = ''
595 tx_mined_status = self.get_tx_height(tx_hash)
596 can_remove = ((tx_mined_status.height in [TX_HEIGHT_FUTURE, TX_HEIGHT_LOCAL])
597 # otherwise 'height' is unreliable (typically LOCAL):
598 and is_relevant
599 # don't offer during common signing flow, e.g. when watch-only wallet starts creating a tx:
600 and bool(tx_we_already_have_in_db))
601 can_dscancel = False
602 if tx.is_complete():
603 if tx_we_already_have_in_db:
604 label = self.get_label_for_txid(tx_hash)
605 if tx_mined_status.height > 0:
606 if tx_mined_status.conf:
607 status = _("{} confirmations").format(tx_mined_status.conf)
608 else:
609 status = _('Not verified')
610 elif tx_mined_status.height in (TX_HEIGHT_UNCONF_PARENT, TX_HEIGHT_UNCONFIRMED):
611 status = _('Unconfirmed')
612 if fee is None:
613 fee = self.get_tx_fee(tx_hash)
614 if fee and self.network and self.config.has_fee_mempool():
615 size = tx.estimated_size()
616 fee_per_byte = fee / size
617 exp_n = self.config.fee_to_depth(fee_per_byte)
618 can_bump = is_any_input_ismine and not tx.is_final()
619 can_dscancel = (is_any_input_ismine and not tx.is_final()
620 and not all([self.is_mine(txout.address) for txout in tx.outputs()]))
621 try:
622 self.cpfp(tx, 0)
623 can_cpfp = True
624 except:
625 can_cpfp = False
626 else:
627 status = _('Local')
628 can_broadcast = self.network is not None
629 can_bump = is_any_input_ismine and not tx.is_final()
630 else:
631 status = _("Signed")
632 can_broadcast = self.network is not None
633 else:
634 assert isinstance(tx, PartialTransaction)
635 s, r = tx.signature_count()
636 status = _("Unsigned") if s == 0 else _('Partially signed') + ' (%d/%d)'%(s,r)
637
638 if is_relevant:
639 if tx_wallet_delta.is_all_input_ismine:
640 assert fee is not None
641 amount = tx_wallet_delta.delta + fee
642 else:
643 amount = tx_wallet_delta.delta
644 else:
645 amount = None
646
647 if is_lightning_funding_tx:
648 can_bump = False # would change txid
649
650 return TxWalletDetails(
651 txid=tx_hash,
652 status=status,
653 label=label,
654 can_broadcast=can_broadcast,
655 can_bump=can_bump,
656 can_cpfp=can_cpfp,
657 can_dscancel=can_dscancel,
658 can_save_as_local=can_save_as_local,
659 amount=amount,
660 fee=fee,
661 tx_mined_status=tx_mined_status,
662 mempool_depth_bytes=exp_n,
663 can_remove=can_remove,
664 is_lightning_funding_tx=is_lightning_funding_tx,
665 )
666
667 def get_spendable_coins(self, domain, *, nonlocal_only=False) -> Sequence[PartialTxInput]:
668 confirmed_only = self.config.get('confirmed_only', False)
669 with self._freeze_lock:
670 frozen_addresses = self._frozen_addresses.copy()
671 utxos = self.get_utxos(domain,
672 excluded_addresses=frozen_addresses,
673 mature_only=True,
674 confirmed_only=confirmed_only,
675 nonlocal_only=nonlocal_only)
676 utxos = [utxo for utxo in utxos if not self.is_frozen_coin(utxo)]
677 return utxos
678
679 @abstractmethod
680 def get_receiving_addresses(self, *, slice_start=None, slice_stop=None) -> Sequence[str]:
681 pass
682
683 @abstractmethod
684 def get_change_addresses(self, *, slice_start=None, slice_stop=None) -> Sequence[str]:
685 pass
686
687 def dummy_address(self):
688 # first receiving address
689 return self.get_receiving_addresses(slice_start=0, slice_stop=1)[0]
690
691 def get_frozen_balance(self):
692 with self._freeze_lock:
693 frozen_addresses = self._frozen_addresses.copy()
694 # note: for coins, use is_frozen_coin instead of _frozen_coins,
695 # as latter only contains *manually* frozen ones
696 frozen_coins = {utxo.prevout.to_str() for utxo in self.get_utxos()
697 if self.is_frozen_coin(utxo)}
698 if not frozen_coins: # shortcut
699 return self.get_balance(frozen_addresses)
700 c1, u1, x1 = self.get_balance()
701 c2, u2, x2 = self.get_balance(
702 excluded_addresses=frozen_addresses,
703 excluded_coins=frozen_coins,
704 )
705 return c1-c2, u1-u2, x1-x2
706
707 def balance_at_timestamp(self, domain, target_timestamp):
708 # we assume that get_history returns items ordered by block height
709 # we also assume that block timestamps are monotonic (which is false...!)
710 h = self.get_history(domain=domain)
711 balance = 0
712 for hist_item in h:
713 balance = hist_item.balance
714 if hist_item.tx_mined_status.timestamp is None or hist_item.tx_mined_status.timestamp > target_timestamp:
715 return balance - hist_item.delta
716 # return last balance
717 return balance
718
719 def get_onchain_history(self, *, domain=None):
720 monotonic_timestamp = 0
721 for hist_item in self.get_history(domain=domain):
722 monotonic_timestamp = max(monotonic_timestamp, (hist_item.tx_mined_status.timestamp or 999_999_999_999))
723 yield {
724 'txid': hist_item.txid,
725 'fee_sat': hist_item.fee,
726 'height': hist_item.tx_mined_status.height,
727 'confirmations': hist_item.tx_mined_status.conf,
728 'timestamp': hist_item.tx_mined_status.timestamp,
729 'monotonic_timestamp': monotonic_timestamp,
730 'incoming': True if hist_item.delta>0 else False,
731 'bc_value': Satoshis(hist_item.delta),
732 'bc_balance': Satoshis(hist_item.balance),
733 'date': timestamp_to_datetime(hist_item.tx_mined_status.timestamp),
734 'label': self.get_label_for_txid(hist_item.txid),
735 'txpos_in_block': hist_item.tx_mined_status.txpos,
736 }
737
738 def create_invoice(self, *, outputs: List[PartialTxOutput], message, pr, URI) -> Invoice:
739 height=self.get_local_height()
740 if pr:
741 return OnchainInvoice.from_bip70_payreq(pr, height)
742 if '!' in (x.value for x in outputs):
743 amount = '!'
744 else:
745 amount = sum(x.value for x in outputs)
746 timestamp = None
747 exp = None
748 if URI:
749 timestamp = URI.get('time')
750 exp = URI.get('exp')
751 timestamp = timestamp or int(time.time())
752 exp = exp or 0
753 _id = bh2u(sha256d(repr(outputs) + "%d"%timestamp))[0:10]
754 invoice = OnchainInvoice(
755 type=PR_TYPE_ONCHAIN,
756 amount_sat=amount,
757 outputs=outputs,
758 message=message,
759 id=_id,
760 time=timestamp,
761 exp=exp,
762 bip70=None,
763 requestor=None,
764 height=height,
765 )
766 return invoice
767
768 def save_invoice(self, invoice: Invoice) -> None:
769 key = self.get_key_for_outgoing_invoice(invoice)
770 if not invoice.is_lightning():
771 assert isinstance(invoice, OnchainInvoice)
772 if self.is_onchain_invoice_paid(invoice, 0):
773 self.logger.info("saving invoice... but it is already paid!")
774 with self.transaction_lock:
775 for txout in invoice.outputs:
776 self._invoices_from_scriptpubkey_map[txout.scriptpubkey].add(key)
777 self.invoices[key] = invoice
778 self.save_db()
779
780 def clear_invoices(self):
781 self.invoices = {}
782 self.save_db()
783
784 def clear_requests(self):
785 self.receive_requests = {}
786 self.save_db()
787
788 def get_invoices(self):
789 out = list(self.invoices.values())
790 out.sort(key=lambda x:x.time)
791 return out
792
793 def get_unpaid_invoices(self):
794 invoices = self.get_invoices()
795 return [x for x in invoices if self.get_invoice_status(x) != PR_PAID]
796
797 def get_invoice(self, key):
798 return self.invoices.get(key)
799
800 def import_requests(self, path):
801 data = read_json_file(path)
802 for x in data:
803 req = Invoice.from_json(x)
804 self.add_payment_request(req)
805
806 def export_requests(self, path):
807 write_json_file(path, list(self.receive_requests.values()))
808
809 def import_invoices(self, path):
810 data = read_json_file(path)
811 for x in data:
812 invoice = Invoice.from_json(x)
813 self.save_invoice(invoice)
814
815 def export_invoices(self, path):
816 write_json_file(path, list(self.invoices.values()))
817
818 def _get_relevant_invoice_keys_for_tx(self, tx: Transaction) -> Set[str]:
819 relevant_invoice_keys = set()
820 with self.transaction_lock:
821 for txout in tx.outputs():
822 for invoice_key in self._invoices_from_scriptpubkey_map.get(txout.scriptpubkey, set()):
823 # note: the invoice might have been deleted since, so check now:
824 if invoice_key in self.invoices:
825 relevant_invoice_keys.add(invoice_key)
826 return relevant_invoice_keys
827
828 def get_relevant_invoices_for_tx(self, tx: Transaction) -> Sequence[OnchainInvoice]:
829 invoice_keys = self._get_relevant_invoice_keys_for_tx(tx)
830 invoices = [self.get_invoice(key) for key in invoice_keys]
831 invoices = [inv for inv in invoices if inv] # filter out None
832 for inv in invoices:
833 assert isinstance(inv, OnchainInvoice), f"unexpected type {type(inv)}"
834 return invoices
835
836 def _prepare_onchain_invoice_paid_detection(self):
837 # scriptpubkey -> list(invoice_keys)
838 self._invoices_from_scriptpubkey_map = defaultdict(set) # type: Dict[bytes, Set[str]]
839 for invoice_key, invoice in self.invoices.items():
840 if invoice.type == PR_TYPE_ONCHAIN:
841 assert isinstance(invoice, OnchainInvoice)
842 for txout in invoice.outputs:
843 self._invoices_from_scriptpubkey_map[txout.scriptpubkey].add(invoice_key)
844
845 def _is_onchain_invoice_paid(self, invoice: Invoice, conf: int) -> Tuple[bool, Sequence[str]]:
846 """Returns whether on-chain invoice is satisfied, and list of relevant TXIDs."""
847 assert invoice.type == PR_TYPE_ONCHAIN
848 assert isinstance(invoice, OnchainInvoice)
849 invoice_amounts = defaultdict(int) # type: Dict[bytes, int] # scriptpubkey -> value_sats
850 for txo in invoice.outputs: # type: PartialTxOutput
851 invoice_amounts[txo.scriptpubkey] += 1 if txo.value == '!' else txo.value
852 relevant_txs = []
853 with self.transaction_lock:
854 for invoice_scriptpubkey, invoice_amt in invoice_amounts.items():
855 scripthash = bitcoin.script_to_scripthash(invoice_scriptpubkey.hex())
856 prevouts_and_values = self.db.get_prevouts_by_scripthash(scripthash)
857 total_received = 0
858 for prevout, v in prevouts_and_values:
859 tx_height = self.get_tx_height(prevout.txid.hex())
860 if tx_height.height > 0 and tx_height.height <= invoice.height:
861 continue
862 if tx_height.conf < conf:
863 continue
864 total_received += v
865 relevant_txs.append(prevout.txid.hex())
866 # check that there is at least one TXO, and that they pay enough.
867 # note: "at least one TXO" check is needed for zero amount invoice (e.g. OP_RETURN)
868 if len(prevouts_and_values) == 0:
869 return False, []
870 if total_received < invoice_amt:
871 return False, []
872 return True, relevant_txs
873
874 def is_onchain_invoice_paid(self, invoice: Invoice, conf: int) -> bool:
875 return self._is_onchain_invoice_paid(invoice, conf)[0]
876
877 def _maybe_set_tx_label_based_on_invoices(self, tx: Transaction) -> bool:
878 # note: this is not done in 'get_default_label' as that would require deserializing each tx
879 tx_hash = tx.txid()
880 labels = []
881 for invoice in self.get_relevant_invoices_for_tx(tx):
882 if invoice.message:
883 labels.append(invoice.message)
884 if labels and not self._labels.get(tx_hash, ''):
885 self.set_label(tx_hash, "; ".join(labels))
886 return bool(labels)
887
888 def add_transaction(self, tx, *, allow_unrelated=False):
889 tx_was_added = super().add_transaction(tx, allow_unrelated=allow_unrelated)
890
891 if tx_was_added:
892 self._maybe_set_tx_label_based_on_invoices(tx)
893 return tx_was_added
894
895 @profiler
896 def get_full_history(self, fx=None, *, onchain_domain=None, include_lightning=True):
897 transactions_tmp = OrderedDictWithIndex()
898 # add on-chain txns
899 onchain_history = self.get_onchain_history(domain=onchain_domain)
900 for tx_item in onchain_history:
901 txid = tx_item['txid']
902 transactions_tmp[txid] = tx_item
903 # add lnworker onchain transactions
904 lnworker_history = self.lnworker.get_onchain_history() if self.lnworker and include_lightning else {}
905 for txid, item in lnworker_history.items():
906 if txid in transactions_tmp:
907 tx_item = transactions_tmp[txid]
908 tx_item['group_id'] = item.get('group_id') # for swaps
909 tx_item['label'] = item['label']
910 tx_item['type'] = item['type']
911 ln_value = Decimal(item['amount_msat']) / 1000 # for channel open/close tx
912 tx_item['ln_value'] = Satoshis(ln_value)
913 else:
914 transactions_tmp[txid] = item
915 ln_value = Decimal(item['amount_msat']) / 1000 # for channel open/close tx
916 item['ln_value'] = Satoshis(ln_value)
917 # add lightning_transactions
918 lightning_history = self.lnworker.get_lightning_history() if self.lnworker and include_lightning else {}
919 for tx_item in lightning_history.values():
920 txid = tx_item.get('txid')
921 ln_value = Decimal(tx_item['amount_msat']) / 1000
922 tx_item['lightning'] = True
923 tx_item['ln_value'] = Satoshis(ln_value)
924 key = tx_item.get('txid') or tx_item['payment_hash']
925 transactions_tmp[key] = tx_item
926 # sort on-chain and LN stuff into new dict, by timestamp
927 # (we rely on this being a *stable* sort)
928 transactions = OrderedDictWithIndex()
929 for k, v in sorted(list(transactions_tmp.items()),
930 key=lambda x: x[1].get('monotonic_timestamp') or x[1].get('timestamp') or float('inf')):
931 transactions[k] = v
932 now = time.time()
933 balance = 0
934 for item in transactions.values():
935 # add on-chain and lightning values
936 value = Decimal(0)
937 if item.get('bc_value'):
938 value += item['bc_value'].value
939 if item.get('ln_value'):
940 value += item.get('ln_value').value
941 # note: 'value' and 'balance' has msat precision (as LN has msat precision)
942 item['value'] = Satoshis(value)
943 balance += value
944 item['balance'] = Satoshis(balance)
945 if fx and fx.is_enabled() and fx.get_history_config():
946 txid = item.get('txid')
947 if not item.get('lightning') and txid:
948 fiat_fields = self.get_tx_item_fiat(tx_hash=txid, amount_sat=value, fx=fx, tx_fee=item['fee_sat'])
949 item.update(fiat_fields)
950 else:
951 timestamp = item['timestamp'] or now
952 fiat_value = value / Decimal(bitcoin.COIN) * fx.timestamp_rate(timestamp)
953 item['fiat_value'] = Fiat(fiat_value, fx.ccy)
954 item['fiat_default'] = True
955 return transactions
956
957 @profiler
958 def get_detailed_history(self, from_timestamp=None, to_timestamp=None,
959 fx=None, show_addresses=False, from_height=None, to_height=None):
960 # History with capital gains, using utxo pricing
961 # FIXME: Lightning capital gains would requires FIFO
962 if (from_timestamp is not None or to_timestamp is not None) \
963 and (from_height is not None or to_height is not None):
964 raise Exception('timestamp and block height based filtering cannot be used together')
965 out = []
966 income = 0
967 expenditures = 0
968 capital_gains = Decimal(0)
969 fiat_income = Decimal(0)
970 fiat_expenditures = Decimal(0)
971 now = time.time()
972 for item in self.get_onchain_history():
973 timestamp = item['timestamp']
974 if from_timestamp and (timestamp or now) < from_timestamp:
975 continue
976 if to_timestamp and (timestamp or now) >= to_timestamp:
977 continue
978 height = item['height']
979 if from_height is not None and from_height > height > 0:
980 continue
981 if to_height is not None and (height >= to_height or height <= 0):
982 continue
983 tx_hash = item['txid']
984 tx = self.db.get_transaction(tx_hash)
985 tx_fee = item['fee_sat']
986 item['fee'] = Satoshis(tx_fee) if tx_fee is not None else None
987 if show_addresses:
988 item['inputs'] = list(map(lambda x: x.to_json(), tx.inputs()))
989 item['outputs'] = list(map(lambda x: {'address': x.get_ui_address_str(), 'value': Satoshis(x.value)},
990 tx.outputs()))
991 # fixme: use in and out values
992 value = item['bc_value'].value
993 if value < 0:
994 expenditures += -value
995 else:
996 income += value
997 # fiat computations
998 if fx and fx.is_enabled() and fx.get_history_config():
999 fiat_fields = self.get_tx_item_fiat(tx_hash=tx_hash, amount_sat=value, fx=fx, tx_fee=tx_fee)
1000 fiat_value = fiat_fields['fiat_value'].value
1001 item.update(fiat_fields)
1002 if value < 0:
1003 capital_gains += fiat_fields['capital_gain'].value
1004 fiat_expenditures += -fiat_value
1005 else:
1006 fiat_income += fiat_value
1007 out.append(item)
1008 # add summary
1009 if out:
1010 b, v = out[0]['bc_balance'].value, out[0]['bc_value'].value
1011 start_balance = None if b is None or v is None else b - v
1012 end_balance = out[-1]['bc_balance'].value
1013 if from_timestamp is not None and to_timestamp is not None:
1014 start_date = timestamp_to_datetime(from_timestamp)
1015 end_date = timestamp_to_datetime(to_timestamp)
1016 else:
1017 start_date = None
1018 end_date = None
1019 summary = {
1020 'start_date': start_date,
1021 'end_date': end_date,
1022 'from_height': from_height,
1023 'to_height': to_height,
1024 'start_balance': Satoshis(start_balance),
1025 'end_balance': Satoshis(end_balance),
1026 'incoming': Satoshis(income),
1027 'outgoing': Satoshis(expenditures)
1028 }
1029 if fx and fx.is_enabled() and fx.get_history_config():
1030 unrealized = self.unrealized_gains(None, fx.timestamp_rate, fx.ccy)
1031 summary['fiat_currency'] = fx.ccy
1032 summary['fiat_capital_gains'] = Fiat(capital_gains, fx.ccy)
1033 summary['fiat_incoming'] = Fiat(fiat_income, fx.ccy)
1034 summary['fiat_outgoing'] = Fiat(fiat_expenditures, fx.ccy)
1035 summary['fiat_unrealized_gains'] = Fiat(unrealized, fx.ccy)
1036 summary['fiat_start_balance'] = Fiat(fx.historical_value(start_balance, start_date), fx.ccy)
1037 summary['fiat_end_balance'] = Fiat(fx.historical_value(end_balance, end_date), fx.ccy)
1038 summary['fiat_start_value'] = Fiat(fx.historical_value(COIN, start_date), fx.ccy)
1039 summary['fiat_end_value'] = Fiat(fx.historical_value(COIN, end_date), fx.ccy)
1040 else:
1041 summary = {}
1042 return {
1043 'transactions': out,
1044 'summary': summary
1045 }
1046
1047 def default_fiat_value(self, tx_hash, fx, value_sat):
1048 return value_sat / Decimal(COIN) * self.price_at_timestamp(tx_hash, fx.timestamp_rate)
1049
1050 def get_tx_item_fiat(
1051 self,
1052 *,
1053 tx_hash: str,
1054 amount_sat: int,
1055 fx: 'FxThread',
1056 tx_fee: Optional[int],
1057 ) -> Dict[str, Any]:
1058 item = {}
1059 fiat_value = self.get_fiat_value(tx_hash, fx.ccy)
1060 fiat_default = fiat_value is None
1061 fiat_rate = self.price_at_timestamp(tx_hash, fx.timestamp_rate)
1062 fiat_value = fiat_value if fiat_value is not None else self.default_fiat_value(tx_hash, fx, amount_sat)
1063 fiat_fee = tx_fee / Decimal(COIN) * fiat_rate if tx_fee is not None else None
1064 item['fiat_currency'] = fx.ccy
1065 item['fiat_rate'] = Fiat(fiat_rate, fx.ccy)
1066 item['fiat_value'] = Fiat(fiat_value, fx.ccy)
1067 item['fiat_fee'] = Fiat(fiat_fee, fx.ccy) if fiat_fee else None
1068 item['fiat_default'] = fiat_default
1069 if amount_sat < 0:
1070 acquisition_price = - amount_sat / Decimal(COIN) * self.average_price(tx_hash, fx.timestamp_rate, fx.ccy)
1071 liquidation_price = - fiat_value
1072 item['acquisition_price'] = Fiat(acquisition_price, fx.ccy)
1073 cg = liquidation_price - acquisition_price
1074 item['capital_gain'] = Fiat(cg, fx.ccy)
1075 return item
1076
1077 def get_label(self, key: str) -> str:
1078 # key is typically: address / txid / LN-payment-hash-hex
1079 return self._labels.get(key) or ''
1080
1081 def get_label_for_txid(self, tx_hash: str) -> str:
1082 return self._labels.get(tx_hash) or self._get_default_label_for_txid(tx_hash)
1083
1084 def _get_default_label_for_txid(self, tx_hash: str) -> str:
1085 # if no inputs are ismine, concat labels of output addresses
1086 if not self.db.get_txi_addresses(tx_hash):
1087 labels = []
1088 for addr in self.db.get_txo_addresses(tx_hash):
1089 label = self._labels.get(addr)
1090 if label:
1091 labels.append(label)
1092 return ', '.join(labels)
1093 return ''
1094
1095 def get_all_labels(self) -> Dict[str, str]:
1096 with self.lock:
1097 return copy.copy(self._labels)
1098
1099 def get_tx_status(self, tx_hash, tx_mined_info: TxMinedInfo):
1100 extra = []
1101 height = tx_mined_info.height
1102 conf = tx_mined_info.conf
1103 timestamp = tx_mined_info.timestamp
1104 if height == TX_HEIGHT_FUTURE:
1105 assert conf < 0, conf
1106 num_blocks_remainining = -conf
1107 return 2, f'in {num_blocks_remainining} blocks'
1108 if conf == 0:
1109 tx = self.db.get_transaction(tx_hash)
1110 if not tx:
1111 return 2, 'unknown'
1112 is_final = tx and tx.is_final()
1113 if not is_final:
1114 extra.append('rbf')
1115 fee = self.get_tx_fee(tx_hash)
1116 if fee is not None:
1117 size = tx.estimated_size()
1118 fee_per_byte = fee / size
1119 extra.append(format_fee_satoshis(fee_per_byte) + ' sat/b')
1120 if fee is not None and height in (TX_HEIGHT_UNCONF_PARENT, TX_HEIGHT_UNCONFIRMED) \
1121 and self.config.has_fee_mempool():
1122 exp_n = self.config.fee_to_depth(fee_per_byte)
1123 if exp_n is not None:
1124 extra.append('%.2f MB'%(exp_n/1000000))
1125 if height == TX_HEIGHT_LOCAL:
1126 status = 3
1127 elif height == TX_HEIGHT_UNCONF_PARENT:
1128 status = 1
1129 elif height == TX_HEIGHT_UNCONFIRMED:
1130 status = 0
1131 else:
1132 status = 2 # not SPV verified
1133 else:
1134 status = 3 + min(conf, 6)
1135 time_str = format_time(timestamp) if timestamp else _("unknown")
1136 status_str = TX_STATUS[status] if status < 4 else time_str
1137 if extra:
1138 status_str += ' [%s]'%(', '.join(extra))
1139 return status, status_str
1140
1141 def relayfee(self):
1142 return relayfee(self.network)
1143
1144 def dust_threshold(self):
1145 return dust_threshold(self.network)
1146
1147 def get_unconfirmed_base_tx_for_batching(self) -> Optional[Transaction]:
1148 candidate = None
1149 for hist_item in self.get_history():
1150 # tx should not be mined yet
1151 if hist_item.tx_mined_status.conf > 0: continue
1152 # conservative future proofing of code: only allow known unconfirmed types
1153 if hist_item.tx_mined_status.height not in (TX_HEIGHT_UNCONFIRMED,
1154 TX_HEIGHT_UNCONF_PARENT,
1155 TX_HEIGHT_LOCAL):
1156 continue
1157 # tx should be "outgoing" from wallet
1158 if hist_item.delta >= 0:
1159 continue
1160 tx = self.db.get_transaction(hist_item.txid)
1161 if not tx:
1162 continue
1163 # is_mine outputs should not be spent yet
1164 # to avoid cancelling our own dependent transactions
1165 txid = tx.txid()
1166 if any([self.is_mine(o.address) and self.db.get_spent_outpoint(txid, output_idx)
1167 for output_idx, o in enumerate(tx.outputs())]):
1168 continue
1169 # all inputs should be is_mine
1170 if not all([self.is_mine(self.get_txin_address(txin)) for txin in tx.inputs()]):
1171 continue
1172 # prefer txns already in mempool (vs local)
1173 if hist_item.tx_mined_status.height == TX_HEIGHT_LOCAL:
1174 candidate = tx
1175 continue
1176 # tx must have opted-in for RBF
1177 if tx.is_final(): continue
1178 return tx
1179 return candidate
1180
1181 def get_change_addresses_for_new_transaction(
1182 self, preferred_change_addr=None, *, allow_reuse: bool = True,
1183 ) -> List[str]:
1184 change_addrs = []
1185 if preferred_change_addr:
1186 if isinstance(preferred_change_addr, (list, tuple)):
1187 change_addrs = list(preferred_change_addr)
1188 else:
1189 change_addrs = [preferred_change_addr]
1190 elif self.use_change:
1191 # Recalc and get unused change addresses
1192 addrs = self.calc_unused_change_addresses()
1193 # New change addresses are created only after a few
1194 # confirmations.
1195 if addrs:
1196 # if there are any unused, select all
1197 change_addrs = addrs
1198 else:
1199 # if there are none, take one randomly from the last few
1200 if not allow_reuse:
1201 return []
1202 addrs = self.get_change_addresses(slice_start=-self.gap_limit_for_change)
1203 change_addrs = [random.choice(addrs)] if addrs else []
1204 for addr in change_addrs:
1205 assert is_address(addr), f"not valid bitcoin address: {addr}"
1206 # note that change addresses are not necessarily ismine
1207 # in which case this is a no-op
1208 self.check_address_for_corruption(addr)
1209 max_change = self.max_change_outputs if self.multiple_change else 1
1210 return change_addrs[:max_change]
1211
1212 def get_single_change_address_for_new_transaction(
1213 self, preferred_change_addr=None, *, allow_reuse: bool = True,
1214 ) -> Optional[str]:
1215 addrs = self.get_change_addresses_for_new_transaction(
1216 preferred_change_addr=preferred_change_addr,
1217 allow_reuse=allow_reuse,
1218 )
1219 if addrs:
1220 return addrs[0]
1221 return None
1222
1223 @check_returned_address_for_corruption
1224 def get_new_sweep_address_for_channel(self) -> str:
1225 # Recalc and get unused change addresses
1226 addrs = self.calc_unused_change_addresses()
1227 if addrs:
1228 selected_addr = addrs[0]
1229 else:
1230 # if there are none, take one randomly from the last few
1231 addrs = self.get_change_addresses(slice_start=-self.gap_limit_for_change)
1232 if addrs:
1233 selected_addr = random.choice(addrs)
1234 else: # fallback for e.g. imported wallets
1235 selected_addr = self.get_receiving_address()
1236 assert is_address(selected_addr), f"not valid bitcoin address: {selected_addr}"
1237 return selected_addr
1238
1239 def make_unsigned_transaction(self, *, coins: Sequence[PartialTxInput],
1240 outputs: List[PartialTxOutput], fee=None,
1241 change_addr: str = None, is_sweep=False) -> PartialTransaction:
1242
1243 if any([c.already_has_some_signatures() for c in coins]):
1244 raise Exception("Some inputs already contain signatures!")
1245
1246 # prevent side-effect with '!'
1247 outputs = copy.deepcopy(outputs)
1248
1249 # check outputs
1250 i_max = None
1251 for i, o in enumerate(outputs):
1252 if o.value == '!':
1253 if i_max is not None:
1254 raise MultipleSpendMaxTxOutputs()
1255 i_max = i
1256
1257 if fee is None and self.config.fee_per_kb() is None:
1258 raise NoDynamicFeeEstimates()
1259
1260 for item in coins:
1261 self.add_input_info(item)
1262
1263 # Fee estimator
1264 if fee is None:
1265 fee_estimator = self.config.estimate_fee
1266 elif isinstance(fee, Number):
1267 fee_estimator = lambda size: fee
1268 elif callable(fee):
1269 fee_estimator = fee
1270 else:
1271 raise Exception(f'Invalid argument fee: {fee}')
1272
1273 if i_max is None:
1274 # Let the coin chooser select the coins to spend
1275 coin_chooser = coinchooser.get_coin_chooser(self.config)
1276 # If there is an unconfirmed RBF tx, merge with it
1277 base_tx = self.get_unconfirmed_base_tx_for_batching()
1278 if self.config.get('batch_rbf', False) and base_tx:
1279 # make sure we don't try to spend change from the tx-to-be-replaced:
1280 coins = [c for c in coins if c.prevout.txid.hex() != base_tx.txid()]
1281 is_local = self.get_tx_height(base_tx.txid()).height == TX_HEIGHT_LOCAL
1282 base_tx = PartialTransaction.from_tx(base_tx)
1283 base_tx.add_info_from_wallet(self)
1284 base_tx_fee = base_tx.get_fee()
1285 relayfeerate = Decimal(self.relayfee()) / 1000
1286 original_fee_estimator = fee_estimator
1287 def fee_estimator(size: Union[int, float, Decimal]) -> int:
1288 size = Decimal(size)
1289 lower_bound = base_tx_fee + round(size * relayfeerate)
1290 lower_bound = lower_bound if not is_local else 0
1291 return int(max(lower_bound, original_fee_estimator(size)))
1292 txi = base_tx.inputs()
1293 txo = list(filter(lambda o: not self.is_change(o.address), base_tx.outputs()))
1294 old_change_addrs = [o.address for o in base_tx.outputs() if self.is_change(o.address)]
1295 else:
1296 txi = []
1297 txo = []
1298 old_change_addrs = []
1299 # change address. if empty, coin_chooser will set it
1300 change_addrs = self.get_change_addresses_for_new_transaction(change_addr or old_change_addrs)
1301 tx = coin_chooser.make_tx(coins=coins,
1302 inputs=txi,
1303 outputs=list(outputs) + txo,
1304 change_addrs=change_addrs,
1305 fee_estimator_vb=fee_estimator,
1306 dust_threshold=self.dust_threshold())
1307 else:
1308 # "spend max" branch
1309 # note: This *will* spend inputs with negative effective value (if there are any).
1310 # Given as the user is spending "max", and so might be abandoning the wallet,
1311 # try to include all UTXOs, otherwise leftover might remain in the UTXO set
1312 # forever. see #5433
1313 # note: Actually it might be the case that not all UTXOs from the wallet are
1314 # being spent if the user manually selected UTXOs.
1315 sendable = sum(map(lambda c: c.value_sats(), coins))
1316 outputs[i_max].value = 0
1317 tx = PartialTransaction.from_io(list(coins), list(outputs))
1318 fee = fee_estimator(tx.estimated_size())
1319 amount = sendable - tx.output_value() - fee
1320 if amount < 0:
1321 raise NotEnoughFunds()
1322 outputs[i_max].value = amount
1323 tx = PartialTransaction.from_io(list(coins), list(outputs))
1324
1325 # Timelock tx to current height.
1326 tx.locktime = get_locktime_for_new_transaction(self.network)
1327
1328 tx.set_rbf(False) # caller can set RBF manually later
1329 tx.add_info_from_wallet(self)
1330 run_hook('make_unsigned_transaction', self, tx)
1331 return tx
1332
1333 def mktx(self, *, outputs: List[PartialTxOutput], password=None, fee=None, change_addr=None,
1334 domain=None, rbf=False, nonlocal_only=False, tx_version=None, sign=True) -> PartialTransaction:
1335 coins = self.get_spendable_coins(domain, nonlocal_only=nonlocal_only)
1336 tx = self.make_unsigned_transaction(coins=coins,
1337 outputs=outputs,
1338 fee=fee,
1339 change_addr=change_addr)
1340 tx.set_rbf(rbf)
1341 if tx_version is not None:
1342 tx.version = tx_version
1343 if sign:
1344 self.sign_transaction(tx, password)
1345 return tx
1346
1347 def is_frozen_address(self, addr: str) -> bool:
1348 return addr in self._frozen_addresses
1349
1350 def is_frozen_coin(self, utxo: PartialTxInput) -> bool:
1351 prevout_str = utxo.prevout.to_str()
1352 frozen = self._frozen_coins.get(prevout_str, None)
1353 # note: there are three possible states for 'frozen':
1354 # True/False if the user explicitly set it,
1355 # None otherwise
1356 if frozen is None:
1357 return self._is_coin_small_and_unconfirmed(utxo)
1358 return bool(frozen)
1359
1360 def _is_coin_small_and_unconfirmed(self, utxo: PartialTxInput) -> bool:
1361 """If true, the coin should not be spent.
1362 The idea here is that an attacker might send us a UTXO in a
1363 large low-fee unconfirmed tx that will ~never confirm. If we
1364 spend it as part of a tx ourselves, that too will not confirm
1365 (unless we use a high fee but that might not be worth it for
1366 a small value UTXO).
1367 In particular, this test triggers for large "dusting transactions"
1368 that are used for advertising purposes by some entities.
1369 see #6960
1370 """
1371 # confirmed UTXOs are fine; check this first for performance:
1372 block_height = utxo.block_height
1373 assert block_height is not None
1374 if block_height > 0:
1375 return False
1376 # exempt large value UTXOs
1377 value_sats = utxo.value_sats()
1378 assert value_sats is not None
1379 threshold = self.config.get('unconf_utxo_freeze_threshold', 5_000)
1380 if value_sats >= threshold:
1381 return False
1382 # if funding tx has any is_mine input, then UTXO is fine
1383 funding_tx = self.db.get_transaction(utxo.prevout.txid.hex())
1384 if funding_tx is None:
1385 # we should typically have the funding tx available;
1386 # might not have it e.g. while not up_to_date
1387 return True
1388 if any(self.is_mine(self.get_txin_address(txin))
1389 for txin in funding_tx.inputs()):
1390 return False
1391 return True
1392
1393 def set_frozen_state_of_addresses(self, addrs: Sequence[str], freeze: bool) -> bool:
1394 """Set frozen state of the addresses to FREEZE, True or False"""
1395 if all(self.is_mine(addr) for addr in addrs):
1396 with self._freeze_lock:
1397 if freeze:
1398 self._frozen_addresses |= set(addrs)
1399 else:
1400 self._frozen_addresses -= set(addrs)
1401 self.db.put('frozen_addresses', list(self._frozen_addresses))
1402 return True
1403 return False
1404
1405 def set_frozen_state_of_coins(self, utxos: Sequence[str], freeze: bool) -> None:
1406 """Set frozen state of the utxos to FREEZE, True or False"""
1407 # basic sanity check that input is not garbage: (see if raises)
1408 [TxOutpoint.from_str(utxo) for utxo in utxos]
1409 with self._freeze_lock:
1410 for utxo in utxos:
1411 self._frozen_coins[utxo] = bool(freeze)
1412
1413 def is_address_reserved(self, addr: str) -> bool:
1414 # note: atm 'reserved' status is only taken into consideration for 'change addresses'
1415 return addr in self._reserved_addresses
1416
1417 def set_reserved_state_of_address(self, addr: str, *, reserved: bool) -> None:
1418 if not self.is_mine(addr):
1419 return
1420 with self.lock:
1421 if reserved:
1422 self._reserved_addresses.add(addr)
1423 else:
1424 self._reserved_addresses.discard(addr)
1425 self.db.put('reserved_addresses', list(self._reserved_addresses))
1426
1427 def can_export(self):
1428 return not self.is_watching_only() and hasattr(self.keystore, 'get_private_key')
1429
1430 def address_is_old(self, address: str, *, req_conf: int = 3) -> bool:
1431 """Returns whether address has any history that is deeply confirmed.
1432 Used for reorg-safe(ish) gap limit roll-forward.
1433 """
1434 max_conf = -1
1435 h = self.db.get_addr_history(address)
1436 needs_spv_check = not self.config.get("skipmerklecheck", False)
1437 for tx_hash, tx_height in h:
1438 if needs_spv_check:
1439 tx_age = self.get_tx_height(tx_hash).conf
1440 else:
1441 if tx_height <= 0:
1442 tx_age = 0
1443 else:
1444 tx_age = self.get_local_height() - tx_height + 1
1445 max_conf = max(max_conf, tx_age)
1446 return max_conf >= req_conf
1447
1448 def bump_fee(
1449 self,
1450 *,
1451 tx: Transaction,
1452 txid: str = None,
1453 new_fee_rate: Union[int, float, Decimal],
1454 coins: Sequence[PartialTxInput] = None,
1455 strategies: Sequence[BumpFeeStrategy] = None,
1456 ) -> PartialTransaction:
1457 """Increase the miner fee of 'tx'.
1458 'new_fee_rate' is the target min rate in sat/vbyte
1459 'coins' is a list of UTXOs we can choose from as potential new inputs to be added
1460 """
1461 txid = txid or tx.txid()
1462 assert txid
1463 assert tx.txid() in (None, txid)
1464 if not isinstance(tx, PartialTransaction):
1465 tx = PartialTransaction.from_tx(tx)
1466 assert isinstance(tx, PartialTransaction)
1467 tx.remove_signatures()
1468 if tx.is_final():
1469 raise CannotBumpFee(_('Transaction is final'))
1470 new_fee_rate = quantize_feerate(new_fee_rate) # strip excess precision
1471 try:
1472 # note: this might download input utxos over network
1473 tx.add_info_from_wallet(self, ignore_network_issues=False)
1474 except NetworkException as e:
1475 raise CannotBumpFee(repr(e))
1476 old_tx_size = tx.estimated_size()
1477 old_fee = tx.get_fee()
1478 assert old_fee is not None
1479 old_fee_rate = old_fee / old_tx_size # sat/vbyte
1480 if new_fee_rate <= old_fee_rate:
1481 raise CannotBumpFee(_("The new fee rate needs to be higher than the old fee rate."))
1482
1483 if not strategies:
1484 strategies = [BumpFeeStrategy.COINCHOOSER, BumpFeeStrategy.DECREASE_CHANGE]
1485 tx_new = None
1486 exc = None
1487 for strat in strategies:
1488 try:
1489 if strat == BumpFeeStrategy.COINCHOOSER:
1490 tx_new = self._bump_fee_through_coinchooser(
1491 tx=tx,
1492 txid=txid,
1493 new_fee_rate=new_fee_rate,
1494 coins=coins,
1495 )
1496 elif strat == BumpFeeStrategy.DECREASE_CHANGE:
1497 tx_new = self._bump_fee_through_decreasing_change(
1498 tx=tx, new_fee_rate=new_fee_rate)
1499 elif strat == BumpFeeStrategy.DECREASE_PAYMENT:
1500 tx_new = self._bump_fee_through_decreasing_payment(
1501 tx=tx, new_fee_rate=new_fee_rate)
1502 else:
1503 raise NotImplementedError(f"unexpected strategy: {strat}")
1504 except CannotBumpFee as e:
1505 exc = e
1506 else:
1507 strat_used = strat
1508 break
1509 if tx_new is None:
1510 assert exc
1511 raise exc # all strategies failed, re-raise last exception
1512
1513 target_min_fee = new_fee_rate * tx_new.estimated_size()
1514 actual_fee = tx_new.get_fee()
1515 if actual_fee + 1 < target_min_fee:
1516 raise CannotBumpFee(
1517 f"bump_fee fee target was not met (strategy: {strat_used}). "
1518 f"got {actual_fee}, expected >={target_min_fee}. "
1519 f"target rate was {new_fee_rate}")
1520 tx_new.locktime = get_locktime_for_new_transaction(self.network)
1521 tx_new.set_rbf(True)
1522 tx_new.add_info_from_wallet(self)
1523 return tx_new
1524
1525 def _bump_fee_through_coinchooser(
1526 self,
1527 *,
1528 tx: PartialTransaction,
1529 txid: str,
1530 new_fee_rate: Union[int, Decimal],
1531 coins: Sequence[PartialTxInput] = None,
1532 ) -> PartialTransaction:
1533 """Increase the miner fee of 'tx'.
1534
1535 - keeps all inputs
1536 - keeps all not is_mine outputs,
1537 - allows adding new inputs
1538 """
1539 assert txid
1540 tx = copy.deepcopy(tx)
1541 tx.add_info_from_wallet(self)
1542 assert tx.get_fee() is not None
1543 old_inputs = list(tx.inputs())
1544 old_outputs = list(tx.outputs())
1545 # change address
1546 old_change_addrs = [o.address for o in old_outputs if self.is_change(o.address)]
1547 change_addrs = self.get_change_addresses_for_new_transaction(old_change_addrs)
1548 # which outputs to keep?
1549 if old_change_addrs:
1550 fixed_outputs = list(filter(lambda o: not self.is_change(o.address), old_outputs))
1551 else:
1552 if all(self.is_mine(o.address) for o in old_outputs):
1553 # all outputs are is_mine and none of them are change.
1554 # we bail out as it's unclear what the user would want!
1555 # the coinchooser bump fee method is probably not a good idea in this case
1556 raise CannotBumpFee(_('All outputs are non-change is_mine'))
1557 old_not_is_mine = list(filter(lambda o: not self.is_mine(o.address), old_outputs))
1558 if old_not_is_mine:
1559 fixed_outputs = old_not_is_mine
1560 else:
1561 fixed_outputs = old_outputs
1562 if not fixed_outputs:
1563 raise CannotBumpFee(_('Could not figure out which outputs to keep'))
1564
1565 if coins is None:
1566 coins = self.get_spendable_coins(None)
1567 # make sure we don't try to spend output from the tx-to-be-replaced:
1568 coins = [c for c in coins if c.prevout.txid.hex() != txid]
1569 for item in coins:
1570 self.add_input_info(item)
1571 def fee_estimator(size):
1572 return self.config.estimate_fee_for_feerate(fee_per_kb=new_fee_rate*1000, size=size)
1573 coin_chooser = coinchooser.get_coin_chooser(self.config)
1574 try:
1575 return coin_chooser.make_tx(
1576 coins=coins,
1577 inputs=old_inputs,
1578 outputs=fixed_outputs,
1579 change_addrs=change_addrs,
1580 fee_estimator_vb=fee_estimator,
1581 dust_threshold=self.dust_threshold())
1582 except NotEnoughFunds as e:
1583 raise CannotBumpFee(e)
1584
1585 def _bump_fee_through_decreasing_change(
1586 self,
1587 *,
1588 tx: PartialTransaction,
1589 new_fee_rate: Union[int, Decimal],
1590 ) -> PartialTransaction:
1591 """Increase the miner fee of 'tx'.
1592
1593 - keeps all inputs
1594 - no new inputs are added
1595 - allows decreasing and removing outputs (change is decreased first)
1596 This is less "safe" than "coinchooser" method as it might end up decreasing
1597 e.g. a payment to a merchant; but e.g. if the user has sent "Max" previously,
1598 this is the only way to RBF.
1599 """
1600 tx = copy.deepcopy(tx)
1601 tx.add_info_from_wallet(self)
1602 assert tx.get_fee() is not None
1603 inputs = tx.inputs()
1604 outputs = tx._outputs # note: we will mutate this directly
1605
1606 # use own outputs
1607 s = list(filter(lambda o: self.is_mine(o.address), outputs))
1608 # ... unless there is none
1609 if not s:
1610 s = outputs
1611 x_fee = run_hook('get_tx_extra_fee', self, tx)
1612 if x_fee:
1613 x_fee_address, x_fee_amount = x_fee
1614 s = list(filter(lambda o: o.address != x_fee_address, s))
1615 if not s:
1616 raise CannotBumpFee('No outputs at all??')
1617
1618 # prioritize low value outputs, to get rid of dust
1619 s = sorted(s, key=lambda o: o.value)
1620 for o in s:
1621 target_fee = int(math.ceil(tx.estimated_size() * new_fee_rate))
1622 delta = target_fee - tx.get_fee()
1623 i = outputs.index(o)
1624 if o.value - delta >= self.dust_threshold():
1625 new_output_value = o.value - delta
1626 assert isinstance(new_output_value, int)
1627 outputs[i].value = new_output_value
1628 delta = 0
1629 break
1630 else:
1631 del outputs[i]
1632 # note: we mutated the outputs of tx, which will affect
1633 # tx.estimated_size() in the next iteration
1634 if delta > 0:
1635 raise CannotBumpFee(_('Could not find suitable outputs'))
1636
1637 return PartialTransaction.from_io(inputs, outputs)
1638
1639 def _bump_fee_through_decreasing_payment(
1640 self,
1641 *,
1642 tx: PartialTransaction,
1643 new_fee_rate: Union[int, Decimal],
1644 ) -> PartialTransaction:
1645 """Increase the miner fee of 'tx'.
1646
1647 - keeps all inputs
1648 - no new inputs are added
1649 - decreases payment outputs (not change!). Each non-ismine output is decreased
1650 proportionally to their byte-size.
1651 """
1652 tx = copy.deepcopy(tx)
1653 tx.add_info_from_wallet(self)
1654 assert tx.get_fee() is not None
1655 inputs = tx.inputs()
1656 outputs = tx.outputs()
1657
1658 # select non-ismine outputs
1659 s = [(idx, out) for (idx, out) in enumerate(outputs)
1660 if not self.is_mine(out.address)]
1661 # exempt 2fa fee output if present
1662 x_fee = run_hook('get_tx_extra_fee', self, tx)
1663 if x_fee:
1664 x_fee_address, x_fee_amount = x_fee
1665 s = [(idx, out) for (idx, out) in s if out.address != x_fee_address]
1666 if not s:
1667 raise CannotBumpFee("Cannot find payment output")
1668
1669 del_out_idxs = set()
1670 tx_size = tx.estimated_size()
1671 cur_fee = tx.get_fee()
1672 # Main loop. Each iteration decreases value of all selected outputs.
1673 # The number of iterations is bounded by len(s) as only the final iteration
1674 # can *not remove* any output.
1675 for __ in range(len(s) + 1):
1676 target_fee = int(math.ceil(tx_size * new_fee_rate))
1677 delta_total = target_fee - cur_fee
1678 if delta_total <= 0:
1679 break
1680 out_size_total = sum(Transaction.estimated_output_size_for_script(out.scriptpubkey.hex())
1681 for (idx, out) in s if idx not in del_out_idxs)
1682 for idx, out in s:
1683 out_size = Transaction.estimated_output_size_for_script(out.scriptpubkey.hex())
1684 delta = int(math.ceil(delta_total * out_size / out_size_total))
1685 if out.value - delta >= self.dust_threshold():
1686 new_output_value = out.value - delta
1687 assert isinstance(new_output_value, int)
1688 outputs[idx].value = new_output_value
1689 cur_fee += delta
1690 else: # remove output
1691 tx_size -= out_size
1692 cur_fee += out.value
1693 del_out_idxs.add(idx)
1694 if delta_total > 0:
1695 raise CannotBumpFee(_('Could not find suitable outputs'))
1696
1697 outputs = [out for (idx, out) in enumerate(outputs) if idx not in del_out_idxs]
1698 return PartialTransaction.from_io(inputs, outputs)
1699
1700 def cpfp(self, tx: Transaction, fee: int) -> Optional[PartialTransaction]:
1701 txid = tx.txid()
1702 for i, o in enumerate(tx.outputs()):
1703 address, value = o.address, o.value
1704 if self.is_mine(address):
1705 break
1706 else:
1707 raise CannotCPFP(_("Could not find suitable output"))
1708 coins = self.get_addr_utxo(address)
1709 item = coins.get(TxOutpoint.from_str(txid+':%d'%i))
1710 if not item:
1711 raise CannotCPFP(_("Could not find coins for output"))
1712 inputs = [item]
1713 out_address = (self.get_single_change_address_for_new_transaction(allow_reuse=False)
1714 or self.get_unused_address()
1715 or address)
1716 output_value = value - fee
1717 if output_value < self.dust_threshold():
1718 raise CannotCPFP(_("The output value remaining after fee is too low."))
1719 outputs = [PartialTxOutput.from_address_and_value(out_address, output_value)]
1720 locktime = get_locktime_for_new_transaction(self.network)
1721 tx_new = PartialTransaction.from_io(inputs, outputs, locktime=locktime)
1722 tx_new.set_rbf(True)
1723 tx_new.add_info_from_wallet(self)
1724 return tx_new
1725
1726 def dscancel(
1727 self, *, tx: Transaction, new_fee_rate: Union[int, float, Decimal]
1728 ) -> PartialTransaction:
1729 """Double-Spend-Cancel: cancel an unconfirmed tx by double-spending
1730 its inputs, paying ourselves.
1731 'new_fee_rate' is the target min rate in sat/vbyte
1732 """
1733 if not isinstance(tx, PartialTransaction):
1734 tx = PartialTransaction.from_tx(tx)
1735 assert isinstance(tx, PartialTransaction)
1736 tx.remove_signatures()
1737
1738 if tx.is_final():
1739 raise CannotDoubleSpendTx(_('Transaction is final'))
1740 new_fee_rate = quantize_feerate(new_fee_rate) # strip excess precision
1741 try:
1742 # note: this might download input utxos over network
1743 tx.add_info_from_wallet(self, ignore_network_issues=False)
1744 except NetworkException as e:
1745 raise CannotDoubleSpendTx(repr(e))
1746 old_tx_size = tx.estimated_size()
1747 old_fee = tx.get_fee()
1748 assert old_fee is not None
1749 old_fee_rate = old_fee / old_tx_size # sat/vbyte
1750 if new_fee_rate <= old_fee_rate:
1751 raise CannotDoubleSpendTx(_("The new fee rate needs to be higher than the old fee rate."))
1752 # grab all ismine inputs
1753 inputs = [txin for txin in tx.inputs()
1754 if self.is_mine(self.get_txin_address(txin))]
1755 value = sum([txin.value_sats() for txin in inputs])
1756 # figure out output address
1757 old_change_addrs = [o.address for o in tx.outputs() if self.is_mine(o.address)]
1758 out_address = (self.get_single_change_address_for_new_transaction(old_change_addrs)
1759 or self.get_receiving_address())
1760 locktime = get_locktime_for_new_transaction(self.network)
1761 outputs = [PartialTxOutput.from_address_and_value(out_address, value)]
1762 tx_new = PartialTransaction.from_io(inputs, outputs, locktime=locktime)
1763 new_tx_size = tx_new.estimated_size()
1764 new_fee = max(
1765 new_fee_rate * new_tx_size,
1766 old_fee + self.relayfee() * new_tx_size / Decimal(1000), # BIP-125 rules 3 and 4
1767 )
1768 new_fee = int(math.ceil(new_fee))
1769 output_value = value - new_fee
1770 if output_value < self.dust_threshold():
1771 raise CannotDoubleSpendTx(_("The output value remaining after fee is too low."))
1772 outputs = [PartialTxOutput.from_address_and_value(out_address, value - new_fee)]
1773 tx_new = PartialTransaction.from_io(inputs, outputs, locktime=locktime)
1774 tx_new.set_rbf(True)
1775 tx_new.add_info_from_wallet(self)
1776 return tx_new
1777
1778 @abstractmethod
1779 def _add_input_sig_info(self, txin: PartialTxInput, address: str, *, only_der_suffix: bool) -> None:
1780 pass
1781
1782 def _add_txinout_derivation_info(self, txinout: Union[PartialTxInput, PartialTxOutput],
1783 address: str, *, only_der_suffix: bool) -> None:
1784 pass # implemented by subclasses
1785
1786 def _add_input_utxo_info(
1787 self,
1788 txin: PartialTxInput,
1789 *,
1790 address: str = None,
1791 ignore_network_issues: bool = True,
1792 ) -> None:
1793 # We prefer to include UTXO (full tx) for every input.
1794 # We cannot include UTXO if the prev tx is not signed yet though (chain of unsigned txs),
1795 # in which case we might include a WITNESS_UTXO.
1796 address = address or txin.address
1797 if txin.witness_utxo is None and txin.is_segwit() and address:
1798 received, spent = self.get_addr_io(address)
1799 item = received.get(txin.prevout.to_str())
1800 if item:
1801 txin_value = item[1]
1802 txin.witness_utxo = TxOutput.from_address_and_value(address, txin_value)
1803 if txin.utxo is None:
1804 txin.utxo = self.get_input_tx(txin.prevout.txid.hex(), ignore_network_issues=ignore_network_issues)
1805 txin.ensure_there_is_only_one_utxo()
1806
1807 def _learn_derivation_path_for_address_from_txinout(self, txinout: Union[PartialTxInput, PartialTxOutput],
1808 address: str) -> bool:
1809 """Tries to learn the derivation path for an address (potentially beyond gap limit)
1810 using data available in given txin/txout.
1811 Returns whether the address was found to be is_mine.
1812 """
1813 return False # implemented by subclasses
1814
1815 def add_input_info(
1816 self,
1817 txin: PartialTxInput,
1818 *,
1819 only_der_suffix: bool = False,
1820 ignore_network_issues: bool = True,
1821 ) -> None:
1822 address = self.get_txin_address(txin)
1823 # note: we add input utxos regardless of is_mine
1824 self._add_input_utxo_info(txin, ignore_network_issues=ignore_network_issues, address=address)
1825 if not self.is_mine(address):
1826 is_mine = self._learn_derivation_path_for_address_from_txinout(txin, address)
1827 if not is_mine:
1828 return
1829 # set script_type first, as later checks might rely on it:
1830 txin.script_type = self.get_txin_type(address)
1831 txin.num_sig = self.m if isinstance(self, Multisig_Wallet) else 1
1832 if txin.redeem_script is None:
1833 try:
1834 redeem_script_hex = self.get_redeem_script(address)
1835 txin.redeem_script = bfh(redeem_script_hex) if redeem_script_hex else None
1836 except UnknownTxinType:
1837 pass
1838 if txin.witness_script is None:
1839 try:
1840 witness_script_hex = self.get_witness_script(address)
1841 txin.witness_script = bfh(witness_script_hex) if witness_script_hex else None
1842 except UnknownTxinType:
1843 pass
1844 self._add_input_sig_info(txin, address, only_der_suffix=only_der_suffix)
1845
1846 def can_sign(self, tx: Transaction) -> bool:
1847 if not isinstance(tx, PartialTransaction):
1848 return False
1849 if tx.is_complete():
1850 return False
1851 # add info to inputs if we can; otherwise we might return a false negative:
1852 tx.add_info_from_wallet(self)
1853 for txin in tx.inputs():
1854 # note: is_mine check needed to avoid false positives.
1855 # just because keystore could sign, txin does not necessarily belong to wallet.
1856 # Example: we have p2pkh-like addresses and txin is a multisig that involves our pubkey.
1857 if not self.is_mine(txin.address):
1858 continue
1859 for k in self.get_keystores():
1860 if k.can_sign_txin(txin):
1861 return True
1862 return False
1863
1864 def get_input_tx(self, tx_hash: str, *, ignore_network_issues=False) -> Optional[Transaction]:
1865 # First look up an input transaction in the wallet where it
1866 # will likely be. If co-signing a transaction it may not have
1867 # all the input txs, in which case we ask the network.
1868 tx = self.db.get_transaction(tx_hash)
1869 if not tx and self.network and self.network.has_internet_connection():
1870 try:
1871 raw_tx = self.network.run_from_another_thread(
1872 self.network.get_transaction(tx_hash, timeout=10))
1873 except NetworkException as e:
1874 self.logger.info(f'got network error getting input txn. err: {repr(e)}. txid: {tx_hash}. '
1875 f'if you are intentionally offline, consider using the --offline flag')
1876 if not ignore_network_issues:
1877 raise e
1878 else:
1879 tx = Transaction(raw_tx)
1880 if not tx and not ignore_network_issues:
1881 raise NetworkException('failed to get prev tx from network')
1882 return tx
1883
1884 def add_output_info(self, txout: PartialTxOutput, *, only_der_suffix: bool = False) -> None:
1885 address = txout.address
1886 if not self.is_mine(address):
1887 is_mine = self._learn_derivation_path_for_address_from_txinout(txout, address)
1888 if not is_mine:
1889 return
1890 txout.script_type = self.get_txin_type(address)
1891 txout.is_mine = True
1892 txout.is_change = self.is_change(address)
1893 if isinstance(self, Multisig_Wallet):
1894 txout.num_sig = self.m
1895 self._add_txinout_derivation_info(txout, address, only_der_suffix=only_der_suffix)
1896 if txout.redeem_script is None:
1897 try:
1898 redeem_script_hex = self.get_redeem_script(address)
1899 txout.redeem_script = bfh(redeem_script_hex) if redeem_script_hex else None
1900 except UnknownTxinType:
1901 pass
1902 if txout.witness_script is None:
1903 try:
1904 witness_script_hex = self.get_witness_script(address)
1905 txout.witness_script = bfh(witness_script_hex) if witness_script_hex else None
1906 except UnknownTxinType:
1907 pass
1908
1909 def sign_transaction(self, tx: Transaction, password) -> Optional[PartialTransaction]:
1910 if self.is_watching_only():
1911 return
1912 if not isinstance(tx, PartialTransaction):
1913 return
1914 # add info to a temporary tx copy; including xpubs
1915 # and full derivation paths as hw keystores might want them
1916 tmp_tx = copy.deepcopy(tx)
1917 tmp_tx.add_info_from_wallet(self, include_xpubs=True)
1918 # sign. start with ready keystores.
1919 for k in sorted(self.get_keystores(), key=lambda ks: ks.ready_to_sign(), reverse=True):
1920 try:
1921 if k.can_sign(tmp_tx):
1922 k.sign_transaction(tmp_tx, password)
1923 except UserCancelled:
1924 continue
1925 # remove sensitive info; then copy back details from temporary tx
1926 tmp_tx.remove_xpubs_and_bip32_paths()
1927 tx.combine_with_other_psbt(tmp_tx)
1928 tx.add_info_from_wallet(self, include_xpubs=False)
1929 return tx
1930
1931 def try_detecting_internal_addresses_corruption(self) -> None:
1932 pass
1933
1934 def check_address_for_corruption(self, addr: str) -> None:
1935 pass
1936
1937 def get_unused_addresses(self) -> Sequence[str]:
1938 domain = self.get_receiving_addresses()
1939 # TODO we should index receive_requests by id
1940 in_use_by_request = [k for k in self.receive_requests.keys()
1941 if self.get_request_status(k) != PR_EXPIRED]
1942 in_use_by_request = set(in_use_by_request)
1943 return [addr for addr in domain if not self.is_used(addr)
1944 and addr not in in_use_by_request]
1945
1946 @check_returned_address_for_corruption
1947 def get_unused_address(self) -> Optional[str]:
1948 """Get an unused receiving address, if there is one.
1949 Note: there might NOT be one available!
1950 """
1951 addrs = self.get_unused_addresses()
1952 if addrs:
1953 return addrs[0]
1954
1955 @check_returned_address_for_corruption
1956 def get_receiving_address(self) -> str:
1957 """Get a receiving address. Guaranteed to always return an address."""
1958 unused_addr = self.get_unused_address()
1959 if unused_addr:
1960 return unused_addr
1961 domain = self.get_receiving_addresses()
1962 if not domain:
1963 raise Exception("no receiving addresses in wallet?!")
1964 choice = domain[0]
1965 for addr in domain:
1966 if not self.is_used(addr):
1967 if addr not in self.receive_requests.keys():
1968 return addr
1969 else:
1970 choice = addr
1971 return choice
1972
1973 def create_new_address(self, for_change: bool = False):
1974 raise Exception("this wallet cannot generate new addresses")
1975
1976 def import_address(self, address: str) -> str:
1977 raise Exception("this wallet cannot import addresses")
1978
1979 def import_addresses(self, addresses: List[str], *,
1980 write_to_disk=True) -> Tuple[List[str], List[Tuple[str, str]]]:
1981 raise Exception("this wallet cannot import addresses")
1982
1983 def delete_address(self, address: str) -> None:
1984 raise Exception("this wallet cannot delete addresses")
1985
1986 def get_onchain_request_status(self, r):
1987 address = r.get_address()
1988 amount = r.get_amount_sat()
1989 received, sent = self.get_addr_io(address)
1990 l = []
1991 for txo, x in received.items():
1992 h, v, is_cb = x
1993 txid, n = txo.split(':')
1994 tx_height = self.get_tx_height(txid)
1995 height = tx_height.height
1996 if height > 0 and height <= r.height:
1997 continue
1998 conf = tx_height.conf
1999 l.append((conf, v))
2000 vsum = 0
2001 for conf, v in reversed(sorted(l)):
2002 vsum += v
2003 if vsum >= amount:
2004 return True, conf
2005 return False, None
2006
2007 def get_request_URI(self, req: OnchainInvoice) -> str:
2008 addr = req.get_address()
2009 message = self.get_label(addr)
2010 amount = req.amount_sat
2011 extra_query_params = {}
2012 if req.time:
2013 extra_query_params['time'] = str(int(req.time))
2014 if req.exp:
2015 extra_query_params['exp'] = str(int(req.exp))
2016 #if req.get('name') and req.get('sig'):
2017 # sig = bfh(req.get('sig'))
2018 # sig = bitcoin.base_encode(sig, base=58)
2019 # extra_query_params['name'] = req['name']
2020 # extra_query_params['sig'] = sig
2021 uri = create_bip21_uri(addr, amount, message, extra_query_params=extra_query_params)
2022 return str(uri)
2023
2024 def check_expired_status(self, r: Invoice, status):
2025 if r.is_lightning() and r.exp == 0:
2026 status = PR_EXPIRED # for BOLT-11 invoices, exp==0 means 0 seconds
2027 if status == PR_UNPAID and r.exp > 0 and r.time + r.exp < time.time():
2028 status = PR_EXPIRED
2029 return status
2030
2031 def get_invoice_status(self, invoice: Invoice):
2032 if invoice.is_lightning():
2033 status = self.lnworker.get_invoice_status(invoice) if self.lnworker else PR_UNKNOWN
2034 else:
2035 if self.is_onchain_invoice_paid(invoice, 1):
2036 status =PR_PAID
2037 elif self.is_onchain_invoice_paid(invoice, 0):
2038 status = PR_UNCONFIRMED
2039 else:
2040 status = PR_UNPAID
2041 return self.check_expired_status(invoice, status)
2042
2043 def get_request_status(self, key):
2044 r = self.get_request(key)
2045 if r is None:
2046 return PR_UNKNOWN
2047 if r.is_lightning():
2048 assert isinstance(r, LNInvoice)
2049 status = self.lnworker.get_payment_status(bfh(r.rhash)) if self.lnworker else PR_UNKNOWN
2050 else:
2051 assert isinstance(r, OnchainInvoice)
2052 paid, conf = self.get_onchain_request_status(r)
2053 if not paid:
2054 status = PR_UNPAID
2055 elif conf == 0:
2056 status = PR_UNCONFIRMED
2057 else:
2058 status = PR_PAID
2059 return self.check_expired_status(r, status)
2060
2061 def get_request(self, key):
2062 return self.receive_requests.get(key)
2063
2064 def get_formatted_request(self, key):
2065 x = self.receive_requests.get(key)
2066 if x:
2067 return self.export_request(x)
2068
2069 def export_request(self, x: Invoice) -> Dict[str, Any]:
2070 key = self.get_key_for_receive_request(x)
2071 status = self.get_request_status(key)
2072 status_str = x.get_status_str(status)
2073 is_lightning = x.is_lightning()
2074 d = {
2075 'is_lightning': is_lightning,
2076 'amount_BTC': format_satoshis(x.get_amount_sat()),
2077 'message': x.message,
2078 'timestamp': x.time,
2079 'expiration': x.exp,
2080 'status': status,
2081 'status_str': status_str,
2082 }
2083 if is_lightning:
2084 assert isinstance(x, LNInvoice)
2085 d['rhash'] = x.rhash
2086 d['invoice'] = x.invoice
2087 d['amount_msat'] = x.get_amount_msat()
2088 if self.lnworker and status == PR_UNPAID:
2089 d['can_receive'] = self.lnworker.can_receive_invoice(x)
2090 else:
2091 assert isinstance(x, OnchainInvoice)
2092 paid, conf = self.get_onchain_request_status(x)
2093 d['amount_sat'] = x.get_amount_sat()
2094 d['address'] = x.get_address()
2095 d['URI'] = self.get_request_URI(x)
2096 if conf is not None:
2097 d['confirmations'] = conf
2098 # add URL if we are running a payserver
2099 payserver = self.config.get_netaddress('payserver_address')
2100 if payserver:
2101 root = self.config.get('payserver_root', '/r')
2102 use_ssl = bool(self.config.get('ssl_keyfile'))
2103 protocol = 'https' if use_ssl else 'http'
2104 base = '%s://%s:%d'%(protocol, payserver.host, payserver.port)
2105 d['view_url'] = base + root + '/pay?id=' + key
2106 if use_ssl and 'URI' in d:
2107 request_url = base + '/bip70/' + key + '.bip70'
2108 d['bip70_url'] = request_url
2109 return d
2110
2111 def export_invoice(self, x: Invoice) -> Dict[str, Any]:
2112 status = self.get_invoice_status(x)
2113 status_str = x.get_status_str(status)
2114 is_lightning = x.is_lightning()
2115 d = {
2116 'is_lightning': is_lightning,
2117 'amount_BTC': format_satoshis(x.get_amount_sat()),
2118 'message': x.message,
2119 'timestamp': x.time,
2120 'expiration': x.exp,
2121 'status': status,
2122 'status_str': status_str,
2123 }
2124 if is_lightning:
2125 assert isinstance(x, LNInvoice)
2126 d['invoice'] = x.invoice
2127 d['amount_msat'] = x.get_amount_msat()
2128 if self.lnworker and status == PR_UNPAID:
2129 d['can_pay'] = self.lnworker.can_pay_invoice(x)
2130 else:
2131 assert isinstance(x, OnchainInvoice)
2132 amount_sat = x.get_amount_sat()
2133 assert isinstance(amount_sat, (int, str, type(None)))
2134 d['amount_sat'] = amount_sat
2135 d['outputs'] = [y.to_legacy_tuple() for y in x.outputs]
2136 if x.bip70:
2137 d['bip70'] = x.bip70
2138 d['requestor'] = x.requestor
2139 return d
2140
2141 def receive_tx_callback(self, tx_hash, tx, tx_height):
2142 super().receive_tx_callback(tx_hash, tx, tx_height)
2143 for txo in tx.outputs():
2144 addr = self.get_txout_address(txo)
2145 if addr in self.receive_requests:
2146 status = self.get_request_status(addr)
2147 util.trigger_callback('request_status', self, addr, status)
2148
2149 def make_payment_request(self, address, amount_sat, message, expiration):
2150 # TODO maybe merge with wallet.create_invoice()...
2151 # note that they use incompatible "id"
2152 amount_sat = amount_sat or 0
2153 timestamp = int(time.time())
2154 _id = bh2u(sha256d(address + "%d"%timestamp))[0:10]
2155 expiration = expiration or 0
2156 return OnchainInvoice(
2157 type=PR_TYPE_ONCHAIN,
2158 outputs=[(TYPE_ADDRESS, address, amount_sat)],
2159 message=message,
2160 time=timestamp,
2161 amount_sat=amount_sat,
2162 exp=expiration,
2163 id=_id,
2164 bip70=None,
2165 requestor=None,
2166 height=self.get_local_height(),
2167 )
2168
2169 def sign_payment_request(self, key, alias, alias_addr, password): # FIXME this is broken
2170 req = self.receive_requests.get(key)
2171 assert isinstance(req, OnchainInvoice)
2172 alias_privkey = self.export_private_key(alias_addr, password)
2173 pr = paymentrequest.make_unsigned_request(req)
2174 paymentrequest.sign_request_with_alias(pr, alias, alias_privkey)
2175 req.bip70 = pr.raw.hex()
2176 req['name'] = pr.pki_data
2177 req['sig'] = bh2u(pr.signature)
2178 self.receive_requests[key] = req
2179
2180 @classmethod
2181 def get_key_for_outgoing_invoice(cls, invoice: Invoice) -> str:
2182 """Return the key to use for this invoice in self.invoices."""
2183 if invoice.is_lightning():
2184 assert isinstance(invoice, LNInvoice)
2185 key = invoice.rhash
2186 else:
2187 assert isinstance(invoice, OnchainInvoice)
2188 key = invoice.id
2189 return key
2190
2191 def get_key_for_receive_request(self, req: Invoice, *, sanity_checks: bool = False) -> str:
2192 """Return the key to use for this invoice in self.receive_requests."""
2193 if not req.is_lightning():
2194 assert isinstance(req, OnchainInvoice)
2195 addr = req.get_address()
2196 if sanity_checks:
2197 if not bitcoin.is_address(addr):
2198 raise Exception(_('Invalid Bitcoin address.'))
2199 if not self.is_mine(addr):
2200 raise Exception(_('Address not in wallet.'))
2201 key = addr
2202 else:
2203 assert isinstance(req, LNInvoice)
2204 key = req.rhash
2205 return key
2206
2207 def add_payment_request(self, req: Invoice):
2208 key = self.get_key_for_receive_request(req, sanity_checks=True)
2209 message = req.message
2210 self.receive_requests[key] = req
2211 self.set_label(key, message) # should be a default label
2212 return req
2213
2214 def delete_request(self, key):
2215 """ lightning or on-chain """
2216 if key in self.receive_requests:
2217 self.remove_payment_request(key)
2218 elif self.lnworker:
2219 self.lnworker.delete_payment(key)
2220
2221 def delete_invoice(self, key):
2222 """ lightning or on-chain """
2223 if key in self.invoices:
2224 self.invoices.pop(key)
2225 elif self.lnworker:
2226 self.lnworker.delete_payment(key)
2227
2228 def remove_payment_request(self, addr):
2229 if addr not in self.receive_requests:
2230 return False
2231 self.receive_requests.pop(addr)
2232 return True
2233
2234 def get_sorted_requests(self) -> List[Invoice]:
2235 """ sorted by timestamp """
2236 out = [self.get_request(x) for x in self.receive_requests.keys()]
2237 out = [x for x in out if x is not None]
2238 out.sort(key=lambda x: x.time)
2239 return out
2240
2241 def get_unpaid_requests(self):
2242 out = [self.get_request(x) for x in self.receive_requests.keys() if self.get_request_status(x) != PR_PAID]
2243 out = [x for x in out if x is not None]
2244 out.sort(key=lambda x: x.time)
2245 return out
2246
2247 @abstractmethod
2248 def get_fingerprint(self) -> str:
2249 """Returns a string that can be used to identify this wallet.
2250 Used e.g. by Labels plugin, and LN channel backups.
2251 Returns empty string "" for wallets that don't have an ID.
2252 """
2253 pass
2254
2255 def can_import_privkey(self):
2256 return False
2257
2258 def can_import_address(self):
2259 return False
2260
2261 def can_delete_address(self):
2262 return False
2263
2264 def has_password(self):
2265 return self.has_keystore_encryption() or self.has_storage_encryption()
2266
2267 def can_have_keystore_encryption(self):
2268 return self.keystore and self.keystore.may_have_password()
2269
2270 def get_available_storage_encryption_version(self) -> StorageEncryptionVersion:
2271 """Returns the type of storage encryption offered to the user.
2272
2273 A wallet file (storage) is either encrypted with this version
2274 or is stored in plaintext.
2275 """
2276 if isinstance(self.keystore, Hardware_KeyStore):
2277 return StorageEncryptionVersion.XPUB_PASSWORD
2278 else:
2279 return StorageEncryptionVersion.USER_PASSWORD
2280
2281 def has_keystore_encryption(self):
2282 """Returns whether encryption is enabled for the keystore.
2283
2284 If True, e.g. signing a transaction will require a password.
2285 """
2286 if self.can_have_keystore_encryption():
2287 return self.db.get('use_encryption', False)
2288 return False
2289
2290 def has_storage_encryption(self):
2291 """Returns whether encryption is enabled for the wallet file on disk."""
2292 return self.storage and self.storage.is_encrypted()
2293
2294 @classmethod
2295 def may_have_password(cls):
2296 return True
2297
2298 def check_password(self, password):
2299 if self.has_keystore_encryption():
2300 self.keystore.check_password(password)
2301 if self.has_storage_encryption():
2302 self.storage.check_password(password)
2303
2304 def update_password(self, old_pw, new_pw, *, encrypt_storage: bool = True):
2305 if old_pw is None and self.has_password():
2306 raise InvalidPassword()
2307 self.check_password(old_pw)
2308 if self.storage:
2309 if encrypt_storage:
2310 enc_version = self.get_available_storage_encryption_version()
2311 else:
2312 enc_version = StorageEncryptionVersion.PLAINTEXT
2313 self.storage.set_password(new_pw, enc_version)
2314 # make sure next storage.write() saves changes
2315 self.db.set_modified(True)
2316
2317 # note: Encrypting storage with a hw device is currently only
2318 # allowed for non-multisig wallets. Further,
2319 # Hardware_KeyStore.may_have_password() == False.
2320 # If these were not the case,
2321 # extra care would need to be taken when encrypting keystores.
2322 self._update_password_for_keystore(old_pw, new_pw)
2323 encrypt_keystore = self.can_have_keystore_encryption()
2324 self.db.set_keystore_encryption(bool(new_pw) and encrypt_keystore)
2325 self.save_db()
2326
2327 @abstractmethod
2328 def _update_password_for_keystore(self, old_pw: Optional[str], new_pw: Optional[str]) -> None:
2329 pass
2330
2331 def sign_message(self, address, message, password):
2332 index = self.get_address_index(address)
2333 return self.keystore.sign_message(index, message, password)
2334
2335 def decrypt_message(self, pubkey: str, message, password) -> bytes:
2336 addr = self.pubkeys_to_address([pubkey])
2337 index = self.get_address_index(addr)
2338 return self.keystore.decrypt_message(index, message, password)
2339
2340 @abstractmethod
2341 def pubkeys_to_address(self, pubkeys: Sequence[str]) -> Optional[str]:
2342 pass
2343
2344 def price_at_timestamp(self, txid, price_func):
2345 """Returns fiat price of bitcoin at the time tx got confirmed."""
2346 timestamp = self.get_tx_height(txid).timestamp
2347 return price_func(timestamp if timestamp else time.time())
2348
2349 def unrealized_gains(self, domain, price_func, ccy):
2350 coins = self.get_utxos(domain)
2351 now = time.time()
2352 p = price_func(now)
2353 ap = sum(self.coin_price(coin.prevout.txid.hex(), price_func, ccy, self.get_txin_value(coin)) for coin in coins)
2354 lp = sum([coin.value_sats() for coin in coins]) * p / Decimal(COIN)
2355 return lp - ap
2356
2357 def average_price(self, txid, price_func, ccy) -> Decimal:
2358 """ Average acquisition price of the inputs of a transaction """
2359 input_value = 0
2360 total_price = 0
2361 txi_addresses = self.db.get_txi_addresses(txid)
2362 if not txi_addresses:
2363 return Decimal('NaN')
2364 for addr in txi_addresses:
2365 d = self.db.get_txi_addr(txid, addr)
2366 for ser, v in d:
2367 input_value += v
2368 total_price += self.coin_price(ser.split(':')[0], price_func, ccy, v)
2369 return total_price / (input_value/Decimal(COIN))
2370
2371 def clear_coin_price_cache(self):
2372 self._coin_price_cache = {}
2373
2374 def coin_price(self, txid, price_func, ccy, txin_value) -> Decimal:
2375 """
2376 Acquisition price of a coin.
2377 This assumes that either all inputs are mine, or no input is mine.
2378 """
2379 if txin_value is None:
2380 return Decimal('NaN')
2381 cache_key = "{}:{}:{}".format(str(txid), str(ccy), str(txin_value))
2382 result = self._coin_price_cache.get(cache_key, None)
2383 if result is not None:
2384 return result
2385 if self.db.get_txi_addresses(txid):
2386 result = self.average_price(txid, price_func, ccy) * txin_value/Decimal(COIN)
2387 self._coin_price_cache[cache_key] = result
2388 return result
2389 else:
2390 fiat_value = self.get_fiat_value(txid, ccy)
2391 if fiat_value is not None:
2392 return fiat_value
2393 else:
2394 p = self.price_at_timestamp(txid, price_func)
2395 return p * txin_value/Decimal(COIN)
2396
2397 def is_billing_address(self, addr):
2398 # overridden for TrustedCoin wallets
2399 return False
2400
2401 @abstractmethod
2402 def is_watching_only(self) -> bool:
2403 pass
2404
2405 def get_keystore(self) -> Optional[KeyStore]:
2406 return self.keystore
2407
2408 def get_keystores(self) -> Sequence[KeyStore]:
2409 return [self.keystore] if self.keystore else []
2410
2411 @abstractmethod
2412 def save_keystore(self):
2413 pass
2414
2415 @abstractmethod
2416 def has_seed(self) -> bool:
2417 pass
2418
2419 @abstractmethod
2420 def get_all_known_addresses_beyond_gap_limit(self) -> Set[str]:
2421 pass
2422
2423 def create_transaction(self, outputs, *, fee=None, feerate=None, change_addr=None, domain_addr=None, domain_coins=None,
2424 unsigned=False, rbf=None, password=None, locktime=None):
2425 if fee is not None and feerate is not None:
2426 raise Exception("Cannot specify both 'fee' and 'feerate' at the same time!")
2427 coins = self.get_spendable_coins(domain_addr)
2428 if domain_coins is not None:
2429 coins = [coin for coin in coins if (coin.prevout.to_str() in domain_coins)]
2430 if feerate is not None:
2431 fee_per_kb = 1000 * Decimal(feerate)
2432 fee_estimator = partial(SimpleConfig.estimate_fee_for_feerate, fee_per_kb)
2433 else:
2434 fee_estimator = fee
2435 tx = self.make_unsigned_transaction(
2436 coins=coins,
2437 outputs=outputs,
2438 fee=fee_estimator,
2439 change_addr=change_addr)
2440 if locktime is not None:
2441 tx.locktime = locktime
2442 if rbf is None:
2443 rbf = bool(self.config.get('use_rbf', True))
2444 tx.set_rbf(rbf)
2445 if not unsigned:
2446 self.sign_transaction(tx, password)
2447 return tx
2448
2449 def get_warning_for_risk_of_burning_coins_as_fees(self, tx: 'PartialTransaction') -> Optional[str]:
2450 """Returns a warning message if there is risk of burning coins as fees if we sign.
2451 Note that if not all inputs are ismine, e.g. coinjoin, the risk is not just about fees.
2452
2453 Note:
2454 - legacy sighash does not commit to any input amounts
2455 - BIP-0143 sighash only commits to the *corresponding* input amount
2456 - BIP-taproot sighash commits to *all* input amounts
2457 """
2458 assert isinstance(tx, PartialTransaction)
2459 # if we have all full previous txs, we *know* all the input amounts -> fine
2460 if all([txin.utxo for txin in tx.inputs()]):
2461 return None
2462 # a single segwit input -> fine
2463 if len(tx.inputs()) == 1 and tx.inputs()[0].is_segwit() and tx.inputs()[0].witness_utxo:
2464 return None
2465 # coinjoin or similar
2466 if any([not self.is_mine(txin.address) for txin in tx.inputs()]):
2467 return (_("Warning") + ": "
2468 + _("The input amounts could not be verified as the previous transactions are missing.\n"
2469 "The amount of money being spent CANNOT be verified."))
2470 # some inputs are legacy
2471 if any([not txin.is_segwit() for txin in tx.inputs()]):
2472 return (_("Warning") + ": "
2473 + _("The fee could not be verified. Signing non-segwit inputs is risky:\n"
2474 "if this transaction was maliciously modified before you sign,\n"
2475 "you might end up paying a higher mining fee than displayed."))
2476 # all inputs are segwit
2477 # https://lists.linuxfoundation.org/pipermail/bitcoin-dev/2017-August/014843.html
2478 return (_("Warning") + ": "
2479 + _("If you received this transaction from an untrusted device, "
2480 "do not accept to sign it more than once,\n"
2481 "otherwise you could end up paying a different fee."))
2482
2483 def get_tx_fee_warning(
2484 self,
2485 *,
2486 invoice_amt: int,
2487 tx_size: int,
2488 fee: int,
2489 ) -> Optional[Tuple[bool, str, str]]:
2490 feerate = Decimal(fee) / tx_size # sat/byte
2491 fee_ratio = Decimal(fee) / invoice_amt if invoice_amt else 1
2492 long_warning = None
2493 short_warning = None
2494 allow_send = True
2495 if feerate < self.relayfee() / 1000:
2496 long_warning = (
2497 _("This transaction requires a higher fee, or it will not be propagated by your current server") + "\n"
2498 + _("Try to raise your transaction fee, or use a server with a lower relay fee.")
2499 )
2500 short_warning = _("below relay fee") + "!"
2501 allow_send = False
2502 elif fee_ratio >= FEE_RATIO_HIGH_WARNING:
2503 long_warning = (
2504 _('Warning') + ': ' + _("The fee for this transaction seems unusually high.")
2505 + f'\n({fee_ratio*100:.2f}% of amount)')
2506 short_warning = _("high fee ratio") + "!"
2507 elif feerate > FEERATE_WARNING_HIGH_FEE / 1000:
2508 long_warning = (
2509 _('Warning') + ': ' + _("The fee for this transaction seems unusually high.")
2510 + f'\n(feerate: {feerate:.2f} sat/byte)')
2511 short_warning = _("high fee rate") + "!"
2512 if long_warning is None:
2513 return None
2514 else:
2515 return allow_send, long_warning, short_warning
2516
2517
2518 class Simple_Wallet(Abstract_Wallet):
2519 # wallet with a single keystore
2520
2521 def is_watching_only(self):
2522 return self.keystore.is_watching_only()
2523
2524 def _update_password_for_keystore(self, old_pw, new_pw):
2525 if self.keystore and self.keystore.may_have_password():
2526 self.keystore.update_password(old_pw, new_pw)
2527 self.save_keystore()
2528
2529 def save_keystore(self):
2530 self.db.put('keystore', self.keystore.dump())
2531
2532 @abstractmethod
2533 def get_public_key(self, address: str) -> Optional[str]:
2534 pass
2535
2536 def get_public_keys(self, address: str) -> Sequence[str]:
2537 return [self.get_public_key(address)]
2538
2539 def get_redeem_script(self, address: str) -> Optional[str]:
2540 txin_type = self.get_txin_type(address)
2541 if txin_type in ('p2pkh', 'p2wpkh', 'p2pk'):
2542 return None
2543 if txin_type == 'p2wpkh-p2sh':
2544 pubkey = self.get_public_key(address)
2545 return bitcoin.p2wpkh_nested_script(pubkey)
2546 if txin_type == 'address':
2547 return None
2548 raise UnknownTxinType(f'unexpected txin_type {txin_type}')
2549
2550 def get_witness_script(self, address: str) -> Optional[str]:
2551 return None
2552
2553
2554 class Imported_Wallet(Simple_Wallet):
2555 # wallet made of imported addresses
2556
2557 wallet_type = 'imported'
2558 txin_type = 'address'
2559
2560 def __init__(self, db, storage, *, config):
2561 Abstract_Wallet.__init__(self, db, storage, config=config)
2562
2563 def is_watching_only(self):
2564 return self.keystore is None
2565
2566 def can_import_privkey(self):
2567 return bool(self.keystore)
2568
2569 def load_keystore(self):
2570 self.keystore = load_keystore(self.db, 'keystore') if self.db.get('keystore') else None
2571
2572 def save_keystore(self):
2573 self.db.put('keystore', self.keystore.dump())
2574
2575 def can_import_address(self):
2576 return self.is_watching_only()
2577
2578 def can_delete_address(self):
2579 return True
2580
2581 def has_seed(self):
2582 return False
2583
2584 def is_deterministic(self):
2585 return False
2586
2587 def is_change(self, address):
2588 return False
2589
2590 def get_all_known_addresses_beyond_gap_limit(self) -> Set[str]:
2591 return set()
2592
2593 def get_fingerprint(self):
2594 return ''
2595
2596 def get_addresses(self):
2597 # note: overridden so that the history can be cleared
2598 return self.db.get_imported_addresses()
2599
2600 def get_receiving_addresses(self, **kwargs):
2601 return self.get_addresses()
2602
2603 def get_change_addresses(self, **kwargs):
2604 return []
2605
2606 def import_addresses(self, addresses: List[str], *,
2607 write_to_disk=True) -> Tuple[List[str], List[Tuple[str, str]]]:
2608 good_addr = [] # type: List[str]
2609 bad_addr = [] # type: List[Tuple[str, str]]
2610 for address in addresses:
2611 if not bitcoin.is_address(address):
2612 bad_addr.append((address, _('invalid address')))
2613 continue
2614 if self.db.has_imported_address(address):
2615 bad_addr.append((address, _('address already in wallet')))
2616 continue
2617 good_addr.append(address)
2618 self.db.add_imported_address(address, {})
2619 self.add_address(address)
2620 if write_to_disk:
2621 self.save_db()
2622 return good_addr, bad_addr
2623
2624 def import_address(self, address: str) -> str:
2625 good_addr, bad_addr = self.import_addresses([address])
2626 if good_addr and good_addr[0] == address:
2627 return address
2628 else:
2629 raise BitcoinException(str(bad_addr[0][1]))
2630
2631 def delete_address(self, address: str) -> None:
2632 if not self.db.has_imported_address(address):
2633 return
2634 if len(self.get_addresses()) <= 1:
2635 raise UserFacingException("cannot delete last remaining address from wallet")
2636 transactions_to_remove = set() # only referred to by this address
2637 transactions_new = set() # txs that are not only referred to by address
2638 with self.lock:
2639 for addr in self.db.get_history():
2640 details = self.get_address_history(addr)
2641 if addr == address:
2642 for tx_hash, height in details:
2643 transactions_to_remove.add(tx_hash)
2644 else:
2645 for tx_hash, height in details:
2646 transactions_new.add(tx_hash)
2647 transactions_to_remove -= transactions_new
2648 self.db.remove_addr_history(address)
2649 for tx_hash in transactions_to_remove:
2650 self.remove_transaction(tx_hash)
2651 self.set_label(address, None)
2652 self.remove_payment_request(address)
2653 self.set_frozen_state_of_addresses([address], False)
2654 pubkey = self.get_public_key(address)
2655 self.db.remove_imported_address(address)
2656 if pubkey:
2657 # delete key iff no other address uses it (e.g. p2pkh and p2wpkh for same key)
2658 for txin_type in bitcoin.WIF_SCRIPT_TYPES.keys():
2659 try:
2660 addr2 = bitcoin.pubkey_to_address(txin_type, pubkey)
2661 except NotImplementedError:
2662 pass
2663 else:
2664 if self.db.has_imported_address(addr2):
2665 break
2666 else:
2667 self.keystore.delete_imported_key(pubkey)
2668 self.save_keystore()
2669 self.save_db()
2670
2671 def is_mine(self, address) -> bool:
2672 if not address: return False
2673 return self.db.has_imported_address(address)
2674
2675 def get_address_index(self, address) -> Optional[str]:
2676 # returns None if address is not mine
2677 return self.get_public_key(address)
2678
2679 def get_address_path_str(self, address):
2680 return None
2681
2682 def get_public_key(self, address) -> Optional[str]:
2683 x = self.db.get_imported_address(address)
2684 return x.get('pubkey') if x else None
2685
2686 def import_private_keys(self, keys: List[str], password: Optional[str], *,
2687 write_to_disk=True) -> Tuple[List[str], List[Tuple[str, str]]]:
2688 good_addr = [] # type: List[str]
2689 bad_keys = [] # type: List[Tuple[str, str]]
2690 for key in keys:
2691 try:
2692 txin_type, pubkey = self.keystore.import_privkey(key, password)
2693 except Exception as e:
2694 bad_keys.append((key, _('invalid private key') + f': {e}'))
2695 continue
2696 if txin_type not in ('p2pkh', 'p2wpkh', 'p2wpkh-p2sh'):
2697 bad_keys.append((key, _('not implemented type') + f': {txin_type}'))
2698 continue
2699 addr = bitcoin.pubkey_to_address(txin_type, pubkey)
2700 good_addr.append(addr)
2701 self.db.add_imported_address(addr, {'type':txin_type, 'pubkey':pubkey})
2702 self.add_address(addr)
2703 self.save_keystore()
2704 if write_to_disk:
2705 self.save_db()
2706 return good_addr, bad_keys
2707
2708 def import_private_key(self, key: str, password: Optional[str]) -> str:
2709 good_addr, bad_keys = self.import_private_keys([key], password=password)
2710 if good_addr:
2711 return good_addr[0]
2712 else:
2713 raise BitcoinException(str(bad_keys[0][1]))
2714
2715 def get_txin_type(self, address):
2716 return self.db.get_imported_address(address).get('type', 'address')
2717
2718 def _add_input_sig_info(self, txin, address, *, only_der_suffix):
2719 if not self.is_mine(address):
2720 return
2721 if txin.script_type in ('unknown', 'address'):
2722 return
2723 elif txin.script_type in ('p2pkh', 'p2wpkh', 'p2wpkh-p2sh'):
2724 pubkey = self.get_public_key(address)
2725 if not pubkey:
2726 return
2727 txin.pubkeys = [bfh(pubkey)]
2728 else:
2729 raise Exception(f'Unexpected script type: {txin.script_type}. '
2730 f'Imported wallets are not implemented to handle this.')
2731
2732 def pubkeys_to_address(self, pubkeys):
2733 pubkey = pubkeys[0]
2734 for addr in self.db.get_imported_addresses(): # FIXME slow...
2735 if self.db.get_imported_address(addr)['pubkey'] == pubkey:
2736 return addr
2737 return None
2738
2739 def decrypt_message(self, pubkey: str, message, password) -> bytes:
2740 # this is significantly faster than the implementation in the superclass
2741 return self.keystore.decrypt_message(pubkey, message, password)
2742
2743
2744 class Deterministic_Wallet(Abstract_Wallet):
2745
2746 def __init__(self, db, storage, *, config):
2747 self._ephemeral_addr_to_addr_index = {} # type: Dict[str, Sequence[int]]
2748 Abstract_Wallet.__init__(self, db, storage, config=config)
2749 self.gap_limit = db.get('gap_limit', 20)
2750 # generate addresses now. note that without libsecp this might block
2751 # for a few seconds!
2752 self.synchronize()
2753
2754 # create lightning keys
2755 if self.can_have_lightning():
2756 self.init_lightning()
2757 ln_xprv = self.db.get('lightning_privkey2')
2758 # lnworker can only be initialized once receiving addresses are available
2759 # therefore we instantiate lnworker in DeterministicWallet
2760 self.lnworker = LNWallet(self, ln_xprv) if ln_xprv else None
2761
2762 def has_seed(self):
2763 return self.keystore.has_seed()
2764
2765 def get_addresses(self):
2766 # note: overridden so that the history can be cleared.
2767 # addresses are ordered based on derivation
2768 out = self.get_receiving_addresses()
2769 out += self.get_change_addresses()
2770 return out
2771
2772 def get_receiving_addresses(self, *, slice_start=None, slice_stop=None):
2773 return self.db.get_receiving_addresses(slice_start=slice_start, slice_stop=slice_stop)
2774
2775 def get_change_addresses(self, *, slice_start=None, slice_stop=None):
2776 return self.db.get_change_addresses(slice_start=slice_start, slice_stop=slice_stop)
2777
2778 @profiler
2779 def try_detecting_internal_addresses_corruption(self):
2780 addresses_all = self.get_addresses()
2781 # sample 1: first few
2782 addresses_sample1 = addresses_all[:10]
2783 # sample2: a few more randomly selected
2784 addresses_rand = addresses_all[10:]
2785 addresses_sample2 = random.sample(addresses_rand, min(len(addresses_rand), 10))
2786 for addr_found in itertools.chain(addresses_sample1, addresses_sample2):
2787 self.check_address_for_corruption(addr_found)
2788
2789 def check_address_for_corruption(self, addr):
2790 if addr and self.is_mine(addr):
2791 if addr != self.derive_address(*self.get_address_index(addr)):
2792 raise InternalAddressCorruption()
2793
2794 def get_seed(self, password):
2795 return self.keystore.get_seed(password)
2796
2797 def change_gap_limit(self, value):
2798 '''This method is not called in the code, it is kept for console use'''
2799 value = int(value)
2800 if value >= self.min_acceptable_gap():
2801 self.gap_limit = value
2802 self.db.put('gap_limit', self.gap_limit)
2803 self.save_db()
2804 return True
2805 else:
2806 return False
2807
2808 def num_unused_trailing_addresses(self, addresses):
2809 k = 0
2810 for addr in addresses[::-1]:
2811 if self.db.get_addr_history(addr):
2812 break
2813 k += 1
2814 return k
2815
2816 def min_acceptable_gap(self) -> int:
2817 # fixme: this assumes wallet is synchronized
2818 n = 0
2819 nmax = 0
2820 addresses = self.get_receiving_addresses()
2821 k = self.num_unused_trailing_addresses(addresses)
2822 for addr in addresses[0:-k]:
2823 if self.address_is_old(addr):
2824 n = 0
2825 else:
2826 n += 1
2827 nmax = max(nmax, n)
2828 return nmax + 1
2829
2830 @abstractmethod
2831 def derive_pubkeys(self, c: int, i: int) -> Sequence[str]:
2832 pass
2833
2834 def derive_address(self, for_change: int, n: int) -> str:
2835 for_change = int(for_change)
2836 pubkeys = self.derive_pubkeys(for_change, n)
2837 return self.pubkeys_to_address(pubkeys)
2838
2839 def export_private_key_for_path(self, path: Union[Sequence[int], str], password: Optional[str]) -> str:
2840 if isinstance(path, str):
2841 path = convert_bip32_path_to_list_of_uint32(path)
2842 pk, compressed = self.keystore.get_private_key(path, password)
2843 txin_type = self.get_txin_type() # assumes no mixed-scripts in wallet
2844 return bitcoin.serialize_privkey(pk, compressed, txin_type)
2845
2846 def get_public_keys_with_deriv_info(self, address: str):
2847 der_suffix = self.get_address_index(address)
2848 der_suffix = [int(x) for x in der_suffix]
2849 return {k.derive_pubkey(*der_suffix): (k, der_suffix)
2850 for k in self.get_keystores()}
2851
2852 def _add_input_sig_info(self, txin, address, *, only_der_suffix):
2853 self._add_txinout_derivation_info(txin, address, only_der_suffix=only_der_suffix)
2854
2855 def _add_txinout_derivation_info(self, txinout, address, *, only_der_suffix):
2856 if not self.is_mine(address):
2857 return
2858 pubkey_deriv_info = self.get_public_keys_with_deriv_info(address)
2859 txinout.pubkeys = sorted([pk for pk in list(pubkey_deriv_info)])
2860 for pubkey in pubkey_deriv_info:
2861 ks, der_suffix = pubkey_deriv_info[pubkey]
2862 fp_bytes, der_full = ks.get_fp_and_derivation_to_be_used_in_partial_tx(der_suffix,
2863 only_der_suffix=only_der_suffix)
2864 txinout.bip32_paths[pubkey] = (fp_bytes, der_full)
2865
2866 def create_new_address(self, for_change: bool = False):
2867 assert type(for_change) is bool
2868 with self.lock:
2869 n = self.db.num_change_addresses() if for_change else self.db.num_receiving_addresses()
2870 address = self.derive_address(int(for_change), n)
2871 self.db.add_change_address(address) if for_change else self.db.add_receiving_address(address)
2872 self.add_address(address)
2873 if for_change:
2874 # note: if it's actually "old", it will get filtered later
2875 self._not_old_change_addresses.append(address)
2876 return address
2877
2878 def synchronize_sequence(self, for_change):
2879 limit = self.gap_limit_for_change if for_change else self.gap_limit
2880 while True:
2881 num_addr = self.db.num_change_addresses() if for_change else self.db.num_receiving_addresses()
2882 if num_addr < limit:
2883 self.create_new_address(for_change)
2884 continue
2885 if for_change:
2886 last_few_addresses = self.get_change_addresses(slice_start=-limit)
2887 else:
2888 last_few_addresses = self.get_receiving_addresses(slice_start=-limit)
2889 if any(map(self.address_is_old, last_few_addresses)):
2890 self.create_new_address(for_change)
2891 else:
2892 break
2893
2894 @AddressSynchronizer.with_local_height_cached
2895 def synchronize(self):
2896 with self.lock:
2897 self.synchronize_sequence(False)
2898 self.synchronize_sequence(True)
2899
2900 def get_all_known_addresses_beyond_gap_limit(self):
2901 # note that we don't stop at first large gap
2902 found = set()
2903
2904 def process_addresses(addrs, gap_limit):
2905 rolling_num_unused = 0
2906 for addr in addrs:
2907 if self.db.get_addr_history(addr):
2908 rolling_num_unused = 0
2909 else:
2910 if rolling_num_unused >= gap_limit:
2911 found.add(addr)
2912 rolling_num_unused += 1
2913
2914 process_addresses(self.get_receiving_addresses(), self.gap_limit)
2915 process_addresses(self.get_change_addresses(), self.gap_limit_for_change)
2916 return found
2917
2918 def get_address_index(self, address) -> Optional[Sequence[int]]:
2919 return self.db.get_address_index(address) or self._ephemeral_addr_to_addr_index.get(address)
2920
2921 def get_address_path_str(self, address):
2922 intpath = self.get_address_index(address)
2923 if intpath is None:
2924 return None
2925 return convert_bip32_intpath_to_strpath(intpath)
2926
2927 def _learn_derivation_path_for_address_from_txinout(self, txinout, address):
2928 for ks in self.get_keystores():
2929 pubkey, der_suffix = ks.find_my_pubkey_in_txinout(txinout, only_der_suffix=True)
2930 if der_suffix is not None:
2931 # note: we already know the pubkey belongs to the keystore,
2932 # but the script template might be different
2933 if len(der_suffix) != 2: continue
2934 try:
2935 my_address = self.derive_address(*der_suffix)
2936 except CannotDerivePubkey:
2937 my_address = None
2938 if my_address == address:
2939 self._ephemeral_addr_to_addr_index[address] = list(der_suffix)
2940 return True
2941 return False
2942
2943 def get_master_public_keys(self):
2944 return [self.get_master_public_key()]
2945
2946 def get_fingerprint(self):
2947 return self.get_master_public_key()
2948
2949 def get_txin_type(self, address=None):
2950 return self.txin_type
2951
2952
2953 class Simple_Deterministic_Wallet(Simple_Wallet, Deterministic_Wallet):
2954
2955 """ Deterministic Wallet with a single pubkey per address """
2956
2957 def __init__(self, db, storage, *, config):
2958 Deterministic_Wallet.__init__(self, db, storage, config=config)
2959
2960 def get_public_key(self, address):
2961 sequence = self.get_address_index(address)
2962 pubkeys = self.derive_pubkeys(*sequence)
2963 return pubkeys[0]
2964
2965 def load_keystore(self):
2966 self.keystore = load_keystore(self.db, 'keystore')
2967 try:
2968 xtype = bip32.xpub_type(self.keystore.xpub)
2969 except:
2970 xtype = 'standard'
2971 self.txin_type = 'p2pkh' if xtype == 'standard' else xtype
2972
2973 def get_master_public_key(self):
2974 return self.keystore.get_master_public_key()
2975
2976 def derive_pubkeys(self, c, i):
2977 return [self.keystore.derive_pubkey(c, i).hex()]
2978
2979
2980
2981
2982
2983
2984 class Standard_Wallet(Simple_Deterministic_Wallet):
2985 wallet_type = 'standard'
2986
2987 def pubkeys_to_address(self, pubkeys):
2988 pubkey = pubkeys[0]
2989 return bitcoin.pubkey_to_address(self.txin_type, pubkey)
2990
2991
2992 class Multisig_Wallet(Deterministic_Wallet):
2993 # generic m of n
2994
2995 def __init__(self, db, storage, *, config):
2996 self.wallet_type = db.get('wallet_type')
2997 self.m, self.n = multisig_type(self.wallet_type)
2998 Deterministic_Wallet.__init__(self, db, storage, config=config)
2999
3000 def get_public_keys(self, address):
3001 return [pk.hex() for pk in self.get_public_keys_with_deriv_info(address)]
3002
3003 def pubkeys_to_address(self, pubkeys):
3004 redeem_script = self.pubkeys_to_scriptcode(pubkeys)
3005 return bitcoin.redeem_script_to_address(self.txin_type, redeem_script)
3006
3007 def pubkeys_to_scriptcode(self, pubkeys: Sequence[str]) -> str:
3008 return transaction.multisig_script(sorted(pubkeys), self.m)
3009
3010 def get_redeem_script(self, address):
3011 txin_type = self.get_txin_type(address)
3012 pubkeys = self.get_public_keys(address)
3013 scriptcode = self.pubkeys_to_scriptcode(pubkeys)
3014 if txin_type == 'p2sh':
3015 return scriptcode
3016 elif txin_type == 'p2wsh-p2sh':
3017 return bitcoin.p2wsh_nested_script(scriptcode)
3018 elif txin_type == 'p2wsh':
3019 return None
3020 raise UnknownTxinType(f'unexpected txin_type {txin_type}')
3021
3022 def get_witness_script(self, address):
3023 txin_type = self.get_txin_type(address)
3024 pubkeys = self.get_public_keys(address)
3025 scriptcode = self.pubkeys_to_scriptcode(pubkeys)
3026 if txin_type == 'p2sh':
3027 return None
3028 elif txin_type in ('p2wsh-p2sh', 'p2wsh'):
3029 return scriptcode
3030 raise UnknownTxinType(f'unexpected txin_type {txin_type}')
3031
3032 def derive_pubkeys(self, c, i):
3033 return [k.derive_pubkey(c, i).hex() for k in self.get_keystores()]
3034
3035 def load_keystore(self):
3036 self.keystores = {}
3037 for i in range(self.n):
3038 name = 'x%d/'%(i+1)
3039 self.keystores[name] = load_keystore(self.db, name)
3040 self.keystore = self.keystores['x1/']
3041 xtype = bip32.xpub_type(self.keystore.xpub)
3042 self.txin_type = 'p2sh' if xtype == 'standard' else xtype
3043
3044 def save_keystore(self):
3045 for name, k in self.keystores.items():
3046 self.db.put(name, k.dump())
3047
3048 def get_keystore(self):
3049 return self.keystores.get('x1/')
3050
3051 def get_keystores(self):
3052 return [self.keystores[i] for i in sorted(self.keystores.keys())]
3053
3054 def can_have_keystore_encryption(self):
3055 return any([k.may_have_password() for k in self.get_keystores()])
3056
3057 def _update_password_for_keystore(self, old_pw, new_pw):
3058 for name, keystore in self.keystores.items():
3059 if keystore.may_have_password():
3060 keystore.update_password(old_pw, new_pw)
3061 self.db.put(name, keystore.dump())
3062
3063 def check_password(self, password):
3064 for name, keystore in self.keystores.items():
3065 if keystore.may_have_password():
3066 keystore.check_password(password)
3067 if self.has_storage_encryption():
3068 self.storage.check_password(password)
3069
3070 def get_available_storage_encryption_version(self):
3071 # multisig wallets are not offered hw device encryption
3072 return StorageEncryptionVersion.USER_PASSWORD
3073
3074 def has_seed(self):
3075 return self.keystore.has_seed()
3076
3077 def is_watching_only(self):
3078 return all([k.is_watching_only() for k in self.get_keystores()])
3079
3080 def get_master_public_key(self):
3081 return self.keystore.get_master_public_key()
3082
3083 def get_master_public_keys(self):
3084 return [k.get_master_public_key() for k in self.get_keystores()]
3085
3086 def get_fingerprint(self):
3087 return ''.join(sorted(self.get_master_public_keys()))
3088
3089
3090 wallet_types = ['standard', 'multisig', 'imported']
3091
3092 def register_wallet_type(category):
3093 wallet_types.append(category)
3094
3095 wallet_constructors = {
3096 'standard': Standard_Wallet,
3097 'old': Standard_Wallet,
3098 'xpub': Standard_Wallet,
3099 'imported': Imported_Wallet
3100 }
3101
3102 def register_constructor(wallet_type, constructor):
3103 wallet_constructors[wallet_type] = constructor
3104
3105 # former WalletFactory
3106 class Wallet(object):
3107 """The main wallet "entry point".
3108 This class is actually a factory that will return a wallet of the correct
3109 type when passed a WalletStorage instance."""
3110
3111 def __new__(self, db: 'WalletDB', storage: Optional[WalletStorage], *, config: SimpleConfig):
3112 wallet_type = db.get('wallet_type')
3113 WalletClass = Wallet.wallet_class(wallet_type)
3114 wallet = WalletClass(db, storage, config=config)
3115 return wallet
3116
3117 @staticmethod
3118 def wallet_class(wallet_type):
3119 if multisig_type(wallet_type):
3120 return Multisig_Wallet
3121 if wallet_type in wallet_constructors:
3122 return wallet_constructors[wallet_type]
3123 raise WalletFileException("Unknown wallet type: " + str(wallet_type))
3124
3125
3126 def create_new_wallet(*, path, config: SimpleConfig, passphrase=None, password=None,
3127 encrypt_file=True, seed_type=None, gap_limit=None) -> dict:
3128 """Create a new wallet"""
3129 storage = WalletStorage(path)
3130 if storage.file_exists():
3131 raise Exception("Remove the existing wallet first!")
3132 db = WalletDB('', manual_upgrades=False)
3133
3134 seed = Mnemonic('en').make_seed(seed_type=seed_type)
3135 k = keystore.from_seed(seed, passphrase)
3136 db.put('keystore', k.dump())
3137 db.put('wallet_type', 'standard')
3138 if gap_limit is not None:
3139 db.put('gap_limit', gap_limit)
3140 wallet = Wallet(db, storage, config=config)
3141 wallet.update_password(old_pw=None, new_pw=password, encrypt_storage=encrypt_file)
3142 wallet.synchronize()
3143 msg = "Please keep your seed in a safe place; if you lose it, you will not be able to restore your wallet."
3144 wallet.save_db()
3145 return {'seed': seed, 'wallet': wallet, 'msg': msg}
3146
3147
3148 def restore_wallet_from_text(text, *, path, config: SimpleConfig,
3149 passphrase=None, password=None, encrypt_file=True,
3150 gap_limit=None) -> dict:
3151 """Restore a wallet from text. Text can be a seed phrase, a master
3152 public key, a master private key, a list of bitcoin addresses
3153 or bitcoin private keys."""
3154 storage = WalletStorage(path)
3155 if storage.file_exists():
3156 raise Exception("Remove the existing wallet first!")
3157 db = WalletDB('', manual_upgrades=False)
3158 text = text.strip()
3159 if keystore.is_address_list(text):
3160 wallet = Imported_Wallet(db, storage, config=config)
3161 addresses = text.split()
3162 good_inputs, bad_inputs = wallet.import_addresses(addresses, write_to_disk=False)
3163 # FIXME tell user about bad_inputs
3164 if not good_inputs:
3165 raise Exception("None of the given addresses can be imported")
3166 elif keystore.is_private_key_list(text, allow_spaces_inside_key=False):
3167 k = keystore.Imported_KeyStore({})
3168 db.put('keystore', k.dump())
3169 wallet = Imported_Wallet(db, storage, config=config)
3170 keys = keystore.get_private_keys(text, allow_spaces_inside_key=False)
3171 good_inputs, bad_inputs = wallet.import_private_keys(keys, None, write_to_disk=False)
3172 # FIXME tell user about bad_inputs
3173 if not good_inputs:
3174 raise Exception("None of the given privkeys can be imported")
3175 else:
3176 if keystore.is_master_key(text):
3177 k = keystore.from_master_key(text)
3178 elif keystore.is_seed(text):
3179 k = keystore.from_seed(text, passphrase)
3180 else:
3181 raise Exception("Seed or key not recognized")
3182 db.put('keystore', k.dump())
3183 db.put('wallet_type', 'standard')
3184 if gap_limit is not None:
3185 db.put('gap_limit', gap_limit)
3186 wallet = Wallet(db, storage, config=config)
3187 assert not storage.file_exists(), "file was created too soon! plaintext keys might have been written to disk"
3188 wallet.update_password(old_pw=None, new_pw=password, encrypt_storage=encrypt_file)
3189 wallet.synchronize()
3190 msg = ("This wallet was restored offline. It may contain more addresses than displayed. "
3191 "Start a daemon and use load_wallet to sync its history.")
3192 wallet.save_db()
3193 return {'wallet': wallet, 'msg': msg}
3194
3195
3196 def check_password_for_directory(config: SimpleConfig, old_password, new_password=None) -> bool:
3197 """Checks password against all wallets and returns True if they can all be updated.
3198 If new_password is not None, update all wallet passwords to new_password.
3199 """
3200 dirname = os.path.dirname(config.get_wallet_path())
3201 failed = []
3202 for filename in os.listdir(dirname):
3203 path = os.path.join(dirname, filename)
3204 if not os.path.isfile(path):
3205 continue
3206 basename = os.path.basename(path)
3207 storage = WalletStorage(path)
3208 if not storage.is_encrypted():
3209 # it is a bit wasteful load the wallet here, but that is fine
3210 # because we are progressively enforcing storage encryption.
3211 db = WalletDB(storage.read(), manual_upgrades=False)
3212 wallet = Wallet(db, storage, config=config)
3213 if wallet.has_keystore_encryption():
3214 try:
3215 wallet.check_password(old_password)
3216 except:
3217 failed.append(basename)
3218 continue
3219 if new_password:
3220 wallet.update_password(old_password, new_password)
3221 else:
3222 if new_password:
3223 wallet.update_password(None, new_password)
3224 continue
3225 if not storage.is_encrypted_with_user_pw():
3226 failed.append(basename)
3227 continue
3228 try:
3229 storage.check_password(old_password)
3230 except:
3231 failed.append(basename)
3232 continue
3233 db = WalletDB(storage.read(), manual_upgrades=False)
3234 wallet = Wallet(db, storage, config=config)
3235 try:
3236 wallet.check_password(old_password)
3237 except:
3238 failed.append(basename)
3239 continue
3240 if new_password:
3241 wallet.update_password(old_password, new_password)
3242 return failed == []
3243
3244
3245 def update_password_for_directory(config: SimpleConfig, old_password, new_password) -> bool:
3246 assert new_password is not None
3247 assert check_password_for_directory(config, old_password, None)
3248 return check_password_for_directory(config, old_password, new_password)