tseparate storage and database (JsonDB) - electrum - Electrum Bitcoin wallet
HTML git clone https://git.parazyd.org/electrum
DIR Log
DIR Files
DIR Refs
DIR Submodules
---
DIR commit 7f2083f667caea442eae3184dadba4e29be7c447
DIR parent d5790ea10994f71d4cd0c01d5a08fe87a1d25b82
HTML Author: ThomasV <thomasv@electrum.org>
Date: Tue, 19 Feb 2019 11:56:46 +0100
separate storage and database (JsonDB)
Diffstat:
M electrum/gui/qt/__init__.py | 2 +-
A electrum/json_db.py | 500 +++++++++++++++++++++++++++++++
M electrum/storage.py | 505 +++----------------------------
M electrum/tests/test_wallet.py | 3 ++-
M electrum/util.py | 11 +++++++++++
M electrum/wallet.py | 3 ++-
6 files changed, 559 insertions(+), 465 deletions(-)
---
DIR diff --git a/electrum/gui/qt/__init__.py b/electrum/gui/qt/__init__.py
t@@ -297,7 +297,7 @@ class ElectrumGui(PrintError):
# Show network dialog if config does not exist
if self.daemon.network:
if self.config.get('auto_connect') is None:
- wizard = InstallWizard(self.config, self.app, self.plugins, None)
+ wizard = InstallWizard(self.config, self.app, self.plugins)
wizard.init_network(self.daemon.network)
wizard.terminate()
DIR diff --git a/electrum/json_db.py b/electrum/json_db.py
t@@ -0,0 +1,500 @@
+#!/usr/bin/env python
+#
+# Electrum - lightweight Bitcoin client
+# Copyright (C) 2015 Thomas Voegtlin
+#
+# Permission is hereby granted, free of charge, to any person
+# obtaining a copy of this software and associated documentation files
+# (the "Software"), to deal in the Software without restriction,
+# including without limitation the rights to use, copy, modify, merge,
+# publish, distribute, sublicense, and/or sell copies of the Software,
+# and to permit persons to whom the Software is furnished to do so,
+# subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+import os
+import ast
+import threading
+import json
+import copy
+import re
+import stat
+import hashlib
+import base64
+import zlib
+from collections import defaultdict
+
+from . import util, bitcoin, ecc
+from .util import PrintError, profiler, InvalidPassword, WalletFileException, bfh, standardize_path, multisig_type
+from .plugin import run_hook, plugin_loaders
+from .keystore import bip44_derivation
+
+
+# seed_version is now used for the version of the wallet file
+
+OLD_SEED_VERSION = 4 # electrum versions < 2.0
+NEW_SEED_VERSION = 11 # electrum versions >= 2.0
+FINAL_SEED_VERSION = 18 # electrum >= 2.7 will set this to prevent
+ # old versions from overwriting new format
+
+
+
+
+
+
+class JsonDB(PrintError):
+
+ def __init__(self, raw, manual_upgrades):
+ self.data = {}
+ self.manual_upgrades = manual_upgrades
+ if raw:
+ self.load_data(raw)
+ else:
+ self.put('seed_version', FINAL_SEED_VERSION)
+
+ def get(self, key, default=None):
+ v = self.data.get(key)
+ if v is None:
+ v = default
+ else:
+ v = copy.deepcopy(v)
+ return v
+
+ def put(self, key, value):
+ try:
+ json.dumps(key, cls=util.MyEncoder)
+ json.dumps(value, cls=util.MyEncoder)
+ except:
+ self.print_error(f"json error: cannot save {repr(key)} ({repr(value)})")
+ return False
+ if value is not None:
+ if self.data.get(key) != value:
+ self.data[key] = copy.deepcopy(value)
+ return True
+ elif key in self.data:
+ self.data.pop(key)
+ return True
+ return False
+
+ def get_all_data(self) -> dict:
+ return copy.deepcopy(self.data)
+
+ def overwrite_all_data(self, data: dict) -> None:
+ try:
+ json.dumps(data, cls=util.MyEncoder)
+ except:
+ self.print_error(f"json error: cannot save {repr(data)}")
+ return
+ with self.db_lock:
+ self.modified = True
+ self.data = copy.deepcopy(data)
+
+ def commit(self):
+ pass
+
+ def dump(self):
+ return json.dumps(self.data, indent=4, sort_keys=True, cls=util.MyEncoder)
+
+ def load_data(self, s):
+ try:
+ self.data = json.loads(s)
+ except:
+ try:
+ d = ast.literal_eval(s)
+ labels = d.get('labels', {})
+ except Exception as e:
+ raise IOError("Cannot read wallet file '%s'" % self.path)
+ self.data = {}
+ for key, value in d.items():
+ try:
+ json.dumps(key)
+ json.dumps(value)
+ except:
+ self.print_error('Failed to convert label to json format', key)
+ continue
+ self.data[key] = value
+ if not isinstance(self.data, dict):
+ raise WalletFileException("Malformed wallet file (not dict)")
+
+ # check here if I need to load a plugin
+ t = self.get('wallet_type')
+ l = plugin_loaders.get(t)
+ if l: l()
+
+ if not self.manual_upgrades:
+ if self.requires_split():
+ raise WalletFileException("This wallet has multiple accounts and must be split")
+ if self.requires_upgrade():
+ self.upgrade()
+
+ def requires_split(self):
+ d = self.get('accounts', {})
+ return len(d) > 1
+
+ def split_accounts(self):
+ result = []
+ # backward compatibility with old wallets
+ d = self.get('accounts', {})
+ if len(d) < 2:
+ return
+ wallet_type = self.get('wallet_type')
+ if wallet_type == 'old':
+ assert len(d) == 2
+ data1 = copy.deepcopy(self.data)
+ data1['accounts'] = {'0': d['0']}
+ data1['suffix'] = 'deterministic'
+ data2 = copy.deepcopy(self.data)
+ data2['accounts'] = {'/x': d['/x']}
+ data2['seed'] = None
+ data2['seed_version'] = None
+ data2['master_public_key'] = None
+ data2['wallet_type'] = 'imported'
+ data2['suffix'] = 'imported'
+ result = [data1, data2]
+
+ elif wallet_type in ['bip44', 'trezor', 'keepkey', 'ledger', 'btchip', 'digitalbitbox', 'safe_t']:
+ mpk = self.get('master_public_keys')
+ for k in d.keys():
+ i = int(k)
+ x = d[k]
+ if x.get("pending"):
+ continue
+ xpub = mpk["x/%d'"%i]
+ new_data = copy.deepcopy(self.data)
+ # save account, derivation and xpub at index 0
+ new_data['accounts'] = {'0': x}
+ new_data['master_public_keys'] = {"x/0'": xpub}
+ new_data['derivation'] = bip44_derivation(k)
+ new_data['suffix'] = k
+ result.append(new_data)
+ else:
+ raise WalletFileException("This wallet has multiple accounts and must be split")
+ return result
+
+ def requires_upgrade(self):
+ return self.get_seed_version() < FINAL_SEED_VERSION
+
+ @profiler
+ def upgrade(self):
+ self.print_error('upgrading wallet format')
+ self.convert_imported()
+ self.convert_wallet_type()
+ self.convert_account()
+ self.convert_version_13_b()
+ self.convert_version_14()
+ self.convert_version_15()
+ self.convert_version_16()
+ self.convert_version_17()
+ self.convert_version_18()
+ self.put('seed_version', FINAL_SEED_VERSION) # just to be sure
+
+ def convert_wallet_type(self):
+ if not self._is_upgrade_method_needed(0, 13):
+ return
+
+ wallet_type = self.get('wallet_type')
+ if wallet_type == 'btchip': wallet_type = 'ledger'
+ if self.get('keystore') or self.get('x1/') or wallet_type=='imported':
+ return False
+ assert not self.requires_split()
+ seed_version = self.get_seed_version()
+ seed = self.get('seed')
+ xpubs = self.get('master_public_keys')
+ xprvs = self.get('master_private_keys', {})
+ mpk = self.get('master_public_key')
+ keypairs = self.get('keypairs')
+ key_type = self.get('key_type')
+ if seed_version == OLD_SEED_VERSION or wallet_type == 'old':
+ d = {
+ 'type': 'old',
+ 'seed': seed,
+ 'mpk': mpk,
+ }
+ self.put('wallet_type', 'standard')
+ self.put('keystore', d)
+
+ elif key_type == 'imported':
+ d = {
+ 'type': 'imported',
+ 'keypairs': keypairs,
+ }
+ self.put('wallet_type', 'standard')
+ self.put('keystore', d)
+
+ elif wallet_type in ['xpub', 'standard']:
+ xpub = xpubs["x/"]
+ xprv = xprvs.get("x/")
+ d = {
+ 'type': 'bip32',
+ 'xpub': xpub,
+ 'xprv': xprv,
+ 'seed': seed,
+ }
+ self.put('wallet_type', 'standard')
+ self.put('keystore', d)
+
+ elif wallet_type in ['bip44']:
+ xpub = xpubs["x/0'"]
+ xprv = xprvs.get("x/0'")
+ d = {
+ 'type': 'bip32',
+ 'xpub': xpub,
+ 'xprv': xprv,
+ }
+ self.put('wallet_type', 'standard')
+ self.put('keystore', d)
+
+ elif wallet_type in ['trezor', 'keepkey', 'ledger', 'digitalbitbox', 'safe_t']:
+ xpub = xpubs["x/0'"]
+ derivation = self.get('derivation', bip44_derivation(0))
+ d = {
+ 'type': 'hardware',
+ 'hw_type': wallet_type,
+ 'xpub': xpub,
+ 'derivation': derivation,
+ }
+ self.put('wallet_type', 'standard')
+ self.put('keystore', d)
+
+ elif (wallet_type == '2fa') or multisig_type(wallet_type):
+ for key in xpubs.keys():
+ d = {
+ 'type': 'bip32',
+ 'xpub': xpubs[key],
+ 'xprv': xprvs.get(key),
+ }
+ if key == 'x1/' and seed:
+ d['seed'] = seed
+ self.put(key, d)
+ else:
+ raise WalletFileException('Unable to tell wallet type. Is this even a wallet file?')
+ # remove junk
+ self.put('master_public_key', None)
+ self.put('master_public_keys', None)
+ self.put('master_private_keys', None)
+ self.put('derivation', None)
+ self.put('seed', None)
+ self.put('keypairs', None)
+ self.put('key_type', None)
+
+ def convert_version_13_b(self):
+ # version 13 is ambiguous, and has an earlier and a later structure
+ if not self._is_upgrade_method_needed(0, 13):
+ return
+
+ if self.get('wallet_type') == 'standard':
+ if self.get('keystore').get('type') == 'imported':
+ pubkeys = self.get('keystore').get('keypairs').keys()
+ d = {'change': []}
+ receiving_addresses = []
+ for pubkey in pubkeys:
+ addr = bitcoin.pubkey_to_address('p2pkh', pubkey)
+ receiving_addresses.append(addr)
+ d['receiving'] = receiving_addresses
+ self.put('addresses', d)
+ self.put('pubkeys', None)
+
+ self.put('seed_version', 13)
+
+ def convert_version_14(self):
+ # convert imported wallets for 3.0
+ if not self._is_upgrade_method_needed(13, 13):
+ return
+
+ if self.get('wallet_type') =='imported':
+ addresses = self.get('addresses')
+ if type(addresses) is list:
+ addresses = dict([(x, None) for x in addresses])
+ self.put('addresses', addresses)
+ elif self.get('wallet_type') == 'standard':
+ if self.get('keystore').get('type')=='imported':
+ addresses = set(self.get('addresses').get('receiving'))
+ pubkeys = self.get('keystore').get('keypairs').keys()
+ assert len(addresses) == len(pubkeys)
+ d = {}
+ for pubkey in pubkeys:
+ addr = bitcoin.pubkey_to_address('p2pkh', pubkey)
+ assert addr in addresses
+ d[addr] = {
+ 'pubkey': pubkey,
+ 'redeem_script': None,
+ 'type': 'p2pkh'
+ }
+ self.put('addresses', d)
+ self.put('pubkeys', None)
+ self.put('wallet_type', 'imported')
+ self.put('seed_version', 14)
+
+ def convert_version_15(self):
+ if not self._is_upgrade_method_needed(14, 14):
+ return
+ if self.get('seed_type') == 'segwit':
+ # should not get here; get_seed_version should have caught this
+ raise Exception('unsupported derivation (development segwit, v14)')
+ self.put('seed_version', 15)
+
+ def convert_version_16(self):
+ # fixes issue #3193 for Imported_Wallets with addresses
+ # also, previous versions allowed importing any garbage as an address
+ # which we now try to remove, see pr #3191
+ if not self._is_upgrade_method_needed(15, 15):
+ return
+
+ def remove_address(addr):
+ def remove_from_dict(dict_name):
+ d = self.get(dict_name, None)
+ if d is not None:
+ d.pop(addr, None)
+ self.put(dict_name, d)
+
+ def remove_from_list(list_name):
+ lst = self.get(list_name, None)
+ if lst is not None:
+ s = set(lst)
+ s -= {addr}
+ self.put(list_name, list(s))
+
+ # note: we don't remove 'addr' from self.get('addresses')
+ remove_from_dict('addr_history')
+ remove_from_dict('labels')
+ remove_from_dict('payment_requests')
+ remove_from_list('frozen_addresses')
+
+ if self.get('wallet_type') == 'imported':
+ addresses = self.get('addresses')
+ assert isinstance(addresses, dict)
+ addresses_new = dict()
+ for address, details in addresses.items():
+ if not bitcoin.is_address(address):
+ remove_address(address)
+ continue
+ if details is None:
+ addresses_new[address] = {}
+ else:
+ addresses_new[address] = details
+ self.put('addresses', addresses_new)
+
+ self.put('seed_version', 16)
+
+ def convert_version_17(self):
+ # delete pruned_txo; construct spent_outpoints
+ if not self._is_upgrade_method_needed(16, 16):
+ return
+
+ self.put('pruned_txo', None)
+
+ from .transaction import Transaction
+ transactions = self.get('transactions', {}) # txid -> raw_tx
+ spent_outpoints = defaultdict(dict)
+ for txid, raw_tx in transactions.items():
+ tx = Transaction(raw_tx)
+ for txin in tx.inputs():
+ if txin['type'] == 'coinbase':
+ continue
+ prevout_hash = txin['prevout_hash']
+ prevout_n = txin['prevout_n']
+ spent_outpoints[prevout_hash][prevout_n] = txid
+ self.put('spent_outpoints', spent_outpoints)
+
+ self.put('seed_version', 17)
+
+ def convert_version_18(self):
+ # delete verified_tx3 as its structure changed
+ if not self._is_upgrade_method_needed(17, 17):
+ return
+ self.put('verified_tx3', None)
+ self.put('seed_version', 18)
+
+ # def convert_version_19(self):
+ # TODO for "next" upgrade:
+ # - move "pw_hash_version" from keystore to storage
+ # pass
+
+ def convert_imported(self):
+ if not self._is_upgrade_method_needed(0, 13):
+ return
+
+ # '/x' is the internal ID for imported accounts
+ d = self.get('accounts', {}).get('/x', {}).get('imported',{})
+ if not d:
+ return False
+ addresses = []
+ keypairs = {}
+ for addr, v in d.items():
+ pubkey, privkey = v
+ if privkey:
+ keypairs[pubkey] = privkey
+ else:
+ addresses.append(addr)
+ if addresses and keypairs:
+ raise WalletFileException('mixed addresses and privkeys')
+ elif addresses:
+ self.put('addresses', addresses)
+ self.put('accounts', None)
+ elif keypairs:
+ self.put('wallet_type', 'standard')
+ self.put('key_type', 'imported')
+ self.put('keypairs', keypairs)
+ self.put('accounts', None)
+ else:
+ raise WalletFileException('no addresses or privkeys')
+
+ def convert_account(self):
+ if not self._is_upgrade_method_needed(0, 13):
+ return
+
+ self.put('accounts', None)
+
+ def _is_upgrade_method_needed(self, min_version, max_version):
+ cur_version = self.get_seed_version()
+ if cur_version > max_version:
+ return False
+ elif cur_version < min_version:
+ raise WalletFileException(
+ 'storage upgrade: unexpected version {} (should be {}-{})'
+ .format(cur_version, min_version, max_version))
+ else:
+ return True
+
+ def get_seed_version(self):
+ seed_version = self.get('seed_version')
+ if not seed_version:
+ seed_version = OLD_SEED_VERSION if len(self.get('master_public_key','')) == 128 else NEW_SEED_VERSION
+ if seed_version > FINAL_SEED_VERSION:
+ raise WalletFileException('This version of Electrum is too old to open this wallet.\n'
+ '(highest supported storage version: {}, version of this file: {})'
+ .format(FINAL_SEED_VERSION, seed_version))
+ if seed_version==14 and self.get('seed_type') == 'segwit':
+ self.raise_unsupported_version(seed_version)
+ if seed_version >=12:
+ return seed_version
+ if seed_version not in [OLD_SEED_VERSION, NEW_SEED_VERSION]:
+ self.raise_unsupported_version(seed_version)
+ return seed_version
+
+ def raise_unsupported_version(self, seed_version):
+ msg = "Your wallet has an unsupported seed version."
+ msg += '\n\nWallet file: %s' % os.path.abspath(self.path)
+ if seed_version in [5, 7, 8, 9, 10, 14]:
+ msg += "\n\nTo open this wallet, try 'git checkout seed_v%d'"%seed_version
+ if seed_version == 6:
+ # version 1.9.8 created v6 wallets when an incorrect seed was entered in the restore dialog
+ msg += '\n\nThis file was created because of a bug in version 1.9.8.'
+ if self.get('master_public_keys') is None and self.get('master_private_keys') is None and self.get('imported_keys') is None:
+ # pbkdf2 (at that time an additional dependency) was not included with the binaries, and wallet creation aborted.
+ msg += "\nIt does not contain any keys, and can safely be removed."
+ else:
+ # creation was complete if electrum was run from source
+ msg += "\nPlease open this file with Electrum 1.9.8, and move your coins to a new wallet."
+ raise WalletFileException(msg)
+
DIR diff --git a/electrum/storage.py b/electrum/storage.py
t@@ -23,9 +23,7 @@
# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
import os
-import ast
import threading
-import json
import copy
import re
import stat
t@@ -39,25 +37,8 @@ from .util import PrintError, profiler, InvalidPassword, WalletFileException, bf
from .plugin import run_hook, plugin_loaders
from .keystore import bip44_derivation
+from .json_db import JsonDB
-# seed_version is now used for the version of the wallet file
-
-OLD_SEED_VERSION = 4 # electrum versions < 2.0
-NEW_SEED_VERSION = 11 # electrum versions >= 2.0
-FINAL_SEED_VERSION = 18 # electrum >= 2.7 will set this to prevent
- # old versions from overwriting new format
-
-
-
-def multisig_type(wallet_type):
- '''If wallet_type is mofn multi-sig, return [m, n],
- otherwise return None.'''
- if not wallet_type:
- return None
- match = re.match(r'(\d+)of(\d+)', wallet_type)
- if match:
- match = [int(x) for x in match.group(1, 2)]
- return match
def get_derivation_used_for_hw_device_encryption():
return ("m"
t@@ -68,39 +49,37 @@ def get_derivation_used_for_hw_device_encryption():
STO_EV_PLAINTEXT, STO_EV_USER_PW, STO_EV_XPUB_PW = range(0, 3)
-class JsonDB(PrintError):
- def __init__(self, path):
+class WalletStorage(PrintError):
+
+ def __init__(self, path, manual_upgrades=False):
self.db_lock = threading.RLock()
- self.data = {}
self.path = standardize_path(path)
self._file_exists = self.path and os.path.exists(self.path)
self.modified = False
- def get(self, key, default=None):
+ DB_Class = JsonDB
+ self.path = path
+ self.print_error("wallet path", self.path)
+ self.pubkey = None
+ if self.file_exists():
+ with open(self.path, "r", encoding='utf-8') as f:
+ self.raw = f.read()
+ self._encryption_version = self._init_encryption_version()
+ if not self.is_encrypted():
+ self.db = DB_Class(self.raw, manual_upgrades)
+ else:
+ self._encryption_version = STO_EV_PLAINTEXT
+ # avoid new wallets getting 'upgraded'
+ self.db = DB_Class('', False)
+
+ def put(self, key,value):
with self.db_lock:
- v = self.data.get(key)
- if v is None:
- v = default
- else:
- v = copy.deepcopy(v)
- return v
+ self.modified |= self.db.put(key, value)
- def put(self, key, value):
- try:
- json.dumps(key, cls=util.MyEncoder)
- json.dumps(value, cls=util.MyEncoder)
- except:
- self.print_error(f"json error: cannot save {repr(key)} ({repr(value)})")
- return
+ def get(self, key, default=None):
with self.db_lock:
- if value is not None:
- if self.data.get(key) != value:
- self.modified = True
- self.data[key] = copy.deepcopy(value)
- elif key in self.data:
- self.modified = True
- self.data.pop(key)
+ return self.db.get(key, default)
@profiler
def write(self):
t@@ -113,9 +92,8 @@ class JsonDB(PrintError):
return
if not self.modified:
return
- s = json.dumps(self.data, indent=4, sort_keys=True, cls=util.MyEncoder)
- s = self.encrypt_before_writing(s)
-
+ self.db.commit()
+ s = self.encrypt_before_writing(self.db.dump())
temp_path = "%s.tmp.%s" % (self.path, os.getpid())
with open(temp_path, "w", encoding='utf-8') as f:
f.write(s)
t@@ -137,57 +115,6 @@ class JsonDB(PrintError):
def file_exists(self):
return self._file_exists
-
-class WalletStorage(JsonDB):
-
- def __init__(self, path, manual_upgrades=False):
- JsonDB.__init__(self, path)
- self.print_error("wallet path", self.path)
- self.manual_upgrades = manual_upgrades
- self.pubkey = None
- if self.file_exists():
- with open(self.path, "r", encoding='utf-8') as f:
- self.raw = f.read()
- self._encryption_version = self._init_encryption_version()
- if not self.is_encrypted():
- self.load_data(self.raw)
- else:
- self._encryption_version = STO_EV_PLAINTEXT
- # avoid new wallets getting 'upgraded'
- self.put('seed_version', FINAL_SEED_VERSION)
-
- def load_data(self, s):
- try:
- self.data = json.loads(s)
- except:
- try:
- d = ast.literal_eval(s)
- labels = d.get('labels', {})
- except Exception as e:
- raise IOError("Cannot read wallet file '%s'" % self.path)
- self.data = {}
- for key, value in d.items():
- try:
- json.dumps(key)
- json.dumps(value)
- except:
- self.print_error('Failed to convert label to json format', key)
- continue
- self.data[key] = value
- if not isinstance(self.data, dict):
- raise WalletFileException("Malformed wallet file (not dict)")
-
- # check here if I need to load a plugin
- t = self.get('wallet_type')
- l = plugin_loaders.get(t)
- if l: l()
-
- if not self.manual_upgrades:
- if self.requires_split():
- raise WalletFileException("This wallet has multiple accounts and must be split")
- if self.requires_upgrade():
- self.upgrade()
-
def is_past_initial_decryption(self):
"""Return if storage is in a usable state for normal operations.
t@@ -254,7 +181,7 @@ class WalletStorage(JsonDB):
s = None
self.pubkey = ec_key.get_public_key_hex()
s = s.decode('utf8')
- self.load_data(s)
+ self.db = JsonDB(s, True)
def encrypt_before_writing(self, plaintext: str) -> str:
s = plaintext
t@@ -292,343 +219,28 @@ class WalletStorage(JsonDB):
with self.db_lock:
self.modified = True
- def requires_split(self):
- d = self.get('accounts', {})
- return len(d) > 1
-
- def split_accounts(storage):
- result = []
- # backward compatibility with old wallets
- d = storage.get('accounts', {})
- if len(d) < 2:
- return
- wallet_type = storage.get('wallet_type')
- if wallet_type == 'old':
- assert len(d) == 2
- storage1 = WalletStorage(storage.path + '.deterministic')
- storage1.data = copy.deepcopy(storage.data)
- storage1.put('accounts', {'0': d['0']})
- storage1.upgrade()
- storage1.write()
- storage2 = WalletStorage(storage.path + '.imported')
- storage2.data = copy.deepcopy(storage.data)
- storage2.put('accounts', {'/x': d['/x']})
- storage2.put('seed', None)
- storage2.put('seed_version', None)
- storage2.put('master_public_key', None)
- storage2.put('wallet_type', 'imported')
- storage2.upgrade()
- storage2.write()
- result = [storage1.path, storage2.path]
- elif wallet_type in ['bip44', 'trezor', 'keepkey', 'ledger', 'btchip', 'digitalbitbox', 'safe_t']:
- mpk = storage.get('master_public_keys')
- for k in d.keys():
- i = int(k)
- x = d[k]
- if x.get("pending"):
- continue
- xpub = mpk["x/%d'"%i]
- new_path = storage.path + '.' + k
- storage2 = WalletStorage(new_path)
- storage2.data = copy.deepcopy(storage.data)
- # save account, derivation and xpub at index 0
- storage2.put('accounts', {'0': x})
- storage2.put('master_public_keys', {"x/0'": xpub})
- storage2.put('derivation', bip44_derivation(k))
- storage2.upgrade()
- storage2.write()
- result.append(new_path)
- else:
- raise WalletFileException("This wallet has multiple accounts and must be split")
- return result
-
def requires_upgrade(self):
- return self.file_exists() and self.get_seed_version() < FINAL_SEED_VERSION
+ return self.db.requires_upgrade()
- @profiler
def upgrade(self):
- self.print_error('upgrading wallet format')
-
- self.convert_imported()
- self.convert_wallet_type()
- self.convert_account()
- self.convert_version_13_b()
- self.convert_version_14()
- self.convert_version_15()
- self.convert_version_16()
- self.convert_version_17()
- self.convert_version_18()
-
- self.put('seed_version', FINAL_SEED_VERSION) # just to be sure
+ self.db.upgrade()
self.write()
- def convert_wallet_type(self):
- if not self._is_upgrade_method_needed(0, 13):
- return
-
- wallet_type = self.get('wallet_type')
- if wallet_type == 'btchip': wallet_type = 'ledger'
- if self.get('keystore') or self.get('x1/') or wallet_type=='imported':
- return False
- assert not self.requires_split()
- seed_version = self.get_seed_version()
- seed = self.get('seed')
- xpubs = self.get('master_public_keys')
- xprvs = self.get('master_private_keys', {})
- mpk = self.get('master_public_key')
- keypairs = self.get('keypairs')
- key_type = self.get('key_type')
- if seed_version == OLD_SEED_VERSION or wallet_type == 'old':
- d = {
- 'type': 'old',
- 'seed': seed,
- 'mpk': mpk,
- }
- self.put('wallet_type', 'standard')
- self.put('keystore', d)
-
- elif key_type == 'imported':
- d = {
- 'type': 'imported',
- 'keypairs': keypairs,
- }
- self.put('wallet_type', 'standard')
- self.put('keystore', d)
-
- elif wallet_type in ['xpub', 'standard']:
- xpub = xpubs["x/"]
- xprv = xprvs.get("x/")
- d = {
- 'type': 'bip32',
- 'xpub': xpub,
- 'xprv': xprv,
- 'seed': seed,
- }
- self.put('wallet_type', 'standard')
- self.put('keystore', d)
-
- elif wallet_type in ['bip44']:
- xpub = xpubs["x/0'"]
- xprv = xprvs.get("x/0'")
- d = {
- 'type': 'bip32',
- 'xpub': xpub,
- 'xprv': xprv,
- }
- self.put('wallet_type', 'standard')
- self.put('keystore', d)
-
- elif wallet_type in ['trezor', 'keepkey', 'ledger', 'digitalbitbox', 'safe_t']:
- xpub = xpubs["x/0'"]
- derivation = self.get('derivation', bip44_derivation(0))
- d = {
- 'type': 'hardware',
- 'hw_type': wallet_type,
- 'xpub': xpub,
- 'derivation': derivation,
- }
- self.put('wallet_type', 'standard')
- self.put('keystore', d)
-
- elif (wallet_type == '2fa') or multisig_type(wallet_type):
- for key in xpubs.keys():
- d = {
- 'type': 'bip32',
- 'xpub': xpubs[key],
- 'xprv': xprvs.get(key),
- }
- if key == 'x1/' and seed:
- d['seed'] = seed
- self.put(key, d)
- else:
- raise WalletFileException('Unable to tell wallet type. Is this even a wallet file?')
- # remove junk
- self.put('master_public_key', None)
- self.put('master_public_keys', None)
- self.put('master_private_keys', None)
- self.put('derivation', None)
- self.put('seed', None)
- self.put('keypairs', None)
- self.put('key_type', None)
-
- def convert_version_13_b(self):
- # version 13 is ambiguous, and has an earlier and a later structure
- if not self._is_upgrade_method_needed(0, 13):
- return
-
- if self.get('wallet_type') == 'standard':
- if self.get('keystore').get('type') == 'imported':
- pubkeys = self.get('keystore').get('keypairs').keys()
- d = {'change': []}
- receiving_addresses = []
- for pubkey in pubkeys:
- addr = bitcoin.pubkey_to_address('p2pkh', pubkey)
- receiving_addresses.append(addr)
- d['receiving'] = receiving_addresses
- self.put('addresses', d)
- self.put('pubkeys', None)
-
- self.put('seed_version', 13)
-
- def convert_version_14(self):
- # convert imported wallets for 3.0
- if not self._is_upgrade_method_needed(13, 13):
- return
-
- if self.get('wallet_type') =='imported':
- addresses = self.get('addresses')
- if type(addresses) is list:
- addresses = dict([(x, None) for x in addresses])
- self.put('addresses', addresses)
- elif self.get('wallet_type') == 'standard':
- if self.get('keystore').get('type')=='imported':
- addresses = set(self.get('addresses').get('receiving'))
- pubkeys = self.get('keystore').get('keypairs').keys()
- assert len(addresses) == len(pubkeys)
- d = {}
- for pubkey in pubkeys:
- addr = bitcoin.pubkey_to_address('p2pkh', pubkey)
- assert addr in addresses
- d[addr] = {
- 'pubkey': pubkey,
- 'redeem_script': None,
- 'type': 'p2pkh'
- }
- self.put('addresses', d)
- self.put('pubkeys', None)
- self.put('wallet_type', 'imported')
- self.put('seed_version', 14)
-
- def convert_version_15(self):
- if not self._is_upgrade_method_needed(14, 14):
- return
- if self.get('seed_type') == 'segwit':
- # should not get here; get_seed_version should have caught this
- raise Exception('unsupported derivation (development segwit, v14)')
- self.put('seed_version', 15)
-
- def convert_version_16(self):
- # fixes issue #3193 for Imported_Wallets with addresses
- # also, previous versions allowed importing any garbage as an address
- # which we now try to remove, see pr #3191
- if not self._is_upgrade_method_needed(15, 15):
- return
-
- def remove_address(addr):
- def remove_from_dict(dict_name):
- d = self.get(dict_name, None)
- if d is not None:
- d.pop(addr, None)
- self.put(dict_name, d)
-
- def remove_from_list(list_name):
- lst = self.get(list_name, None)
- if lst is not None:
- s = set(lst)
- s -= {addr}
- self.put(list_name, list(s))
-
- # note: we don't remove 'addr' from self.get('addresses')
- remove_from_dict('addr_history')
- remove_from_dict('labels')
- remove_from_dict('payment_requests')
- remove_from_list('frozen_addresses')
-
- if self.get('wallet_type') == 'imported':
- addresses = self.get('addresses')
- assert isinstance(addresses, dict)
- addresses_new = dict()
- for address, details in addresses.items():
- if not bitcoin.is_address(address):
- remove_address(address)
- continue
- if details is None:
- addresses_new[address] = {}
- else:
- addresses_new[address] = details
- self.put('addresses', addresses_new)
-
- self.put('seed_version', 16)
-
- def convert_version_17(self):
- # delete pruned_txo; construct spent_outpoints
- if not self._is_upgrade_method_needed(16, 16):
- return
-
- self.put('pruned_txo', None)
-
- from .transaction import Transaction
- transactions = self.get('transactions', {}) # txid -> raw_tx
- spent_outpoints = defaultdict(dict)
- for txid, raw_tx in transactions.items():
- tx = Transaction(raw_tx)
- for txin in tx.inputs():
- if txin['type'] == 'coinbase':
- continue
- prevout_hash = txin['prevout_hash']
- prevout_n = txin['prevout_n']
- spent_outpoints[prevout_hash][prevout_n] = txid
- self.put('spent_outpoints', spent_outpoints)
-
- self.put('seed_version', 17)
-
- def convert_version_18(self):
- # delete verified_tx3 as its structure changed
- if not self._is_upgrade_method_needed(17, 17):
- return
- self.put('verified_tx3', None)
- self.put('seed_version', 18)
-
- # def convert_version_19(self):
- # TODO for "next" upgrade:
- # - move "pw_hash_version" from keystore to storage
- # pass
-
- def convert_imported(self):
- if not self._is_upgrade_method_needed(0, 13):
- return
-
- # '/x' is the internal ID for imported accounts
- d = self.get('accounts', {}).get('/x', {}).get('imported',{})
- if not d:
- return False
- addresses = []
- keypairs = {}
- for addr, v in d.items():
- pubkey, privkey = v
- if privkey:
- keypairs[pubkey] = privkey
- else:
- addresses.append(addr)
- if addresses and keypairs:
- raise WalletFileException('mixed addresses and privkeys')
- elif addresses:
- self.put('addresses', addresses)
- self.put('accounts', None)
- elif keypairs:
- self.put('wallet_type', 'standard')
- self.put('key_type', 'imported')
- self.put('keypairs', keypairs)
- self.put('accounts', None)
- else:
- raise WalletFileException('no addresses or privkeys')
-
- def convert_account(self):
- if not self._is_upgrade_method_needed(0, 13):
- return
-
- self.put('accounts', None)
-
- def _is_upgrade_method_needed(self, min_version, max_version):
- cur_version = self.get_seed_version()
- if cur_version > max_version:
- return False
- elif cur_version < min_version:
- raise WalletFileException(
- 'storage upgrade: unexpected version {} (should be {}-{})'
- .format(cur_version, min_version, max_version))
- else:
- return True
+ def requires_split(self):
+ return self.db.requires_split()
+
+ def split_accounts(self):
+ out = []
+ result = self.db.split_accounts()
+ for data in result:
+ path = self.path + '.' + data['suffix']
+ storage = WalletStorage(path)
+ storage.db.data = data
+ storage.db.upgrade()
+ storage.modified = True
+ storage.write()
+ out.append(path)
+ return out
def get_action(self):
action = run_hook('get_action', self)
t@@ -641,34 +253,3 @@ class WalletStorage(JsonDB):
if not self.file_exists():
return 'new'
- def get_seed_version(self):
- seed_version = self.get('seed_version')
- if not seed_version:
- seed_version = OLD_SEED_VERSION if len(self.get('master_public_key','')) == 128 else NEW_SEED_VERSION
- if seed_version > FINAL_SEED_VERSION:
- raise WalletFileException('This version of Electrum is too old to open this wallet.\n'
- '(highest supported storage version: {}, version of this file: {})'
- .format(FINAL_SEED_VERSION, seed_version))
- if seed_version==14 and self.get('seed_type') == 'segwit':
- self.raise_unsupported_version(seed_version)
- if seed_version >=12:
- return seed_version
- if seed_version not in [OLD_SEED_VERSION, NEW_SEED_VERSION]:
- self.raise_unsupported_version(seed_version)
- return seed_version
-
- def raise_unsupported_version(self, seed_version):
- msg = "Your wallet has an unsupported seed version."
- msg += '\n\nWallet file: %s' % os.path.abspath(self.path)
- if seed_version in [5, 7, 8, 9, 10, 14]:
- msg += "\n\nTo open this wallet, try 'git checkout seed_v%d'"%seed_version
- if seed_version == 6:
- # version 1.9.8 created v6 wallets when an incorrect seed was entered in the restore dialog
- msg += '\n\nThis file was created because of a bug in version 1.9.8.'
- if self.get('master_public_keys') is None and self.get('master_private_keys') is None and self.get('imported_keys') is None:
- # pbkdf2 (at that time an additional dependency) was not included with the binaries, and wallet creation aborted.
- msg += "\nIt does not contain any keys, and can safely be removed."
- else:
- # creation was complete if electrum was run from source
- msg += "\nPlease open this file with Electrum 1.9.8, and move your coins to a new wallet."
- raise WalletFileException(msg)
DIR diff --git a/electrum/tests/test_wallet.py b/electrum/tests/test_wallet.py
t@@ -8,7 +8,8 @@ from unittest import TestCase
import time
from io import StringIO
-from electrum.storage import WalletStorage, FINAL_SEED_VERSION
+from electrum.storage import WalletStorage
+from electrum.json_db import FINAL_SEED_VERSION
from electrum.wallet import Abstract_Wallet
from electrum.exchange_rate import ExchangeBase, FxThread
from electrum.util import TxMinedInfo
DIR diff --git a/electrum/util.py b/electrum/util.py
t@@ -1114,3 +1114,14 @@ class OrderedDictWithIndex(OrderedDict):
self._key_to_pos[key] = pos
self._pos_to_key[pos] = key
return ret
+
+
+def multisig_type(wallet_type):
+ '''If wallet_type is mofn multi-sig, return [m, n],
+ otherwise return None.'''
+ if not wallet_type:
+ return None
+ match = re.match(r'(\d+)of(\d+)', wallet_type)
+ if match:
+ match = [int(x) for x in match.group(1, 2)]
+ return match
DIR diff --git a/electrum/wallet.py b/electrum/wallet.py
t@@ -50,7 +50,8 @@ from .bitcoin import (COIN, TYPE_ADDRESS, is_address, address_to_script,
is_minikey, relayfee, dust_threshold)
from .crypto import sha256d
from .keystore import load_keystore, Hardware_KeyStore
-from .storage import multisig_type, STO_EV_PLAINTEXT, STO_EV_USER_PW, STO_EV_XPUB_PW, WalletStorage
+from .util import multisig_type
+from .storage import STO_EV_PLAINTEXT, STO_EV_USER_PW, STO_EV_XPUB_PW, WalletStorage
from . import transaction, bitcoin, coinchooser, paymentrequest, ecc, bip32
from .transaction import Transaction, TxOutput, TxOutputHwInfo
from .plugin import run_hook