tcommon.py - electrum-personal-server - Maximally lightweight electrum server for a single user
HTML git clone https://git.parazyd.org/electrum-personal-server
DIR Log
DIR Files
DIR Refs
DIR README
---
tcommon.py (23598B)
---
1 import socket
2 import time
3 from datetime import datetime
4 import ssl
5 import os
6 import os.path
7 import logging
8 import tempfile
9 import platform
10 import json
11 import traceback
12 from json.decoder import JSONDecodeError
13 from configparser import RawConfigParser, NoSectionError, NoOptionError
14 from ipaddress import ip_network, ip_address
15
16 from electrumpersonalserver.server.jsonrpc import JsonRpc, JsonRpcError
17 import electrumpersonalserver.server.hashes as hashes
18 import electrumpersonalserver.server.deterministicwallet as deterministicwallet
19 import electrumpersonalserver.server.transactionmonitor as transactionmonitor
20 from electrumpersonalserver.server.electrumprotocol import (
21 SERVER_VERSION_NUMBER,
22 UnknownScripthashError,
23 ElectrumProtocol,
24 get_block_header,
25 get_current_header,
26 get_block_headers_hex,
27 DONATION_ADDR,
28 )
29 from electrumpersonalserver.server.mempoolhistogram import (
30 MempoolSync,
31 PollIntervalChange
32 )
33
34 ##python has demented rules for variable scope, so these
35 ## global variables are actually mutable lists
36 bestblockhash = [None]
37
38 last_heartbeat_listening = [datetime.now()]
39 last_heartbeat_connected = [datetime.now()]
40
41 def on_heartbeat_listening(poll_interval_listening, txmonitor):
42 if ((datetime.now() - last_heartbeat_listening[0]).total_seconds()
43 < poll_interval_listening):
44 return True
45 last_heartbeat_listening[0] = datetime.now()
46 logger = logging.getLogger('ELECTRUMPERSONALSERVER')
47 try:
48 txmonitor.check_for_updated_txes()
49 is_node_reachable = True
50 except JsonRpcError:
51 is_node_reachable = False
52 return is_node_reachable
53
54 def on_heartbeat_connected(poll_interval_connected, rpc, txmonitor, protocol):
55 if ((datetime.now() - last_heartbeat_connected[0]).total_seconds()
56 < poll_interval_connected):
57 return
58 last_heartbeat_connected[0] = datetime.now()
59 logger = logging.getLogger('ELECTRUMPERSONALSERVER')
60 is_tip_updated, header = check_for_new_blockchain_tip(rpc,
61 protocol.are_headers_raw)
62 if is_tip_updated:
63 logger.debug("Blockchain tip updated " + (str(header["height"]) if
64 "height" in header else ""))
65 protocol.on_blockchain_tip_updated(header)
66 updated_scripthashes = txmonitor.check_for_updated_txes()
67 protocol.on_updated_scripthashes(updated_scripthashes)
68
69 def check_for_new_blockchain_tip(rpc, raw):
70 new_bestblockhash, header = get_current_header(rpc, raw)
71 is_tip_new = bestblockhash[0] != new_bestblockhash
72 bestblockhash[0] = new_bestblockhash
73 return is_tip_new, header
74
75 def create_server_socket(hostport):
76 logger = logging.getLogger('ELECTRUMPERSONALSERVER')
77 server_sock = socket.socket()
78 server_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
79 server_sock.bind(hostport)
80 server_sock.listen(1)
81 logger.info("Listening for Electrum Wallet on " + str(hostport) + "\n\n"
82 + "If this project is valuable to you please consider donating:\n\t"
83 + DONATION_ADDR)
84 return server_sock
85
86 def run_electrum_server(rpc, txmonitor, config):
87 logger = logging.getLogger('ELECTRUMPERSONALSERVER')
88 logger.debug("Starting electrum server")
89
90 hostport = (config.get("electrum-server", "host"),
91 int(config.get("electrum-server", "port")))
92 ip_whitelist = []
93 for ip in config.get("electrum-server", "ip_whitelist").split(" "):
94 if ip == "*":
95 #matches everything
96 ip_whitelist.append(ip_network("0.0.0.0/0"))
97 ip_whitelist.append(ip_network("::0/0"))
98 else:
99 ip_whitelist.append(ip_network(ip, strict=False))
100 poll_interval_listening = int(config.get("bitcoin-rpc",
101 "poll_interval_listening"))
102 poll_interval_connected = int(config.get("bitcoin-rpc",
103 "poll_interval_connected"))
104 certfile, keyfile = get_certs(config)
105 logger.debug('using cert: {}, key: {}'.format(certfile, keyfile))
106 disable_mempool_fee_histogram = config.getboolean("electrum-server",
107 "disable_mempool_fee_histogram", fallback=False)
108 mempool_update_interval = int(config.get("bitcoin-rpc",
109 "mempool_update_interval", fallback=60))
110 broadcast_method = config.get("electrum-server", "broadcast_method",
111 fallback="own-node")
112 tor_host = config.get("electrum-server", "tor_host", fallback="localhost")
113 tor_port = int(config.get("electrum-server", "tor_port", fallback="9050"))
114 tor_hostport = (tor_host, tor_port)
115
116 mempool_sync = MempoolSync(rpc,
117 disable_mempool_fee_histogram, mempool_update_interval)
118 mempool_sync.initial_sync(logger)
119
120 protocol = ElectrumProtocol(rpc, txmonitor, logger, broadcast_method,
121 tor_hostport, mempool_sync)
122
123 normal_listening_timeout = min(poll_interval_listening,
124 mempool_update_interval)
125 fast_listening_timeout = 0.5
126 server_sock = create_server_socket(hostport)
127 server_sock.settimeout(normal_listening_timeout)
128 accepting_clients = True
129 while True:
130 # main server loop, runs forever
131 sock = None
132 while sock == None:
133 # loop waiting for a successful connection from client
134 try:
135 sock, addr = server_sock.accept()
136 if not accepting_clients:
137 logger.debug("Refusing connection from client because"
138 + " Bitcoin node isnt reachable")
139 raise ConnectionRefusedError()
140 if not any([ip_address(addr[0]) in ipnet
141 for ipnet in ip_whitelist]):
142 logger.debug(addr[0] + " not in whitelist, closing")
143 raise ConnectionRefusedError()
144 sock = ssl.wrap_socket(sock, server_side=True,
145 certfile=certfile, keyfile=keyfile,
146 ssl_version=ssl.PROTOCOL_SSLv23)
147 except socket.timeout:
148 poll_interval_change = mempool_sync.poll_update(1)
149 if poll_interval_change == PollIntervalChange.FAST_POLLING:
150 server_sock.settimeout(fast_listening_timeout)
151 elif poll_interval_change == PollIntervalChange.NORMAL_POLLING:
152 server_sock.settimeout(normal_listening_timeout)
153
154 is_node_reachable = on_heartbeat_listening(
155 poll_interval_listening, txmonitor)
156 accepting_clients = is_node_reachable
157 except (ConnectionRefusedError, ssl.SSLError, IOError):
158 sock.close()
159 sock = None
160 logger.debug('Electrum connected from ' + str(addr[0]))
161
162 def send_reply_fun(reply):
163 line = json.dumps(reply)
164 sock.sendall(line.encode('utf-8') + b'\n')
165 logger.debug('<= ' + line)
166 protocol.set_send_reply_fun(send_reply_fun)
167
168 try:
169 normal_connected_timeout = min(poll_interval_connected,
170 mempool_update_interval)
171 fast_connected_timeout = 0.5
172 sock.settimeout(normal_connected_timeout)
173 recv_buffer = bytearray()
174 while True:
175 # loop for replying to client queries
176 try:
177 recv_data = sock.recv(4096)
178 if not recv_data or len(recv_data) == 0:
179 raise EOFError()
180 recv_buffer.extend(recv_data)
181 lb = recv_buffer.find(b'\n')
182 if lb == -1:
183 continue
184 while lb != -1:
185 line = recv_buffer[:lb].rstrip()
186 recv_buffer = recv_buffer[lb + 1:]
187 lb = recv_buffer.find(b'\n')
188 try:
189 line = line.decode("utf-8")
190 query = json.loads(line)
191 except (UnicodeDecodeError, JSONDecodeError) as e:
192 raise IOError(repr(e))
193 logger.debug("=> " + line)
194 protocol.handle_query(query)
195 except socket.timeout:
196 poll_interval_change = mempool_sync.poll_update(1)
197 if poll_interval_change == PollIntervalChange.FAST_POLLING:
198 sock.settimeout(fast_connected_timeout)
199 elif (poll_interval_change
200 == PollIntervalChange.NORMAL_POLLING):
201 sock.settimeout(normal_connected_timeout)
202
203 on_heartbeat_connected(poll_interval_connected, rpc,
204 txmonitor, protocol)
205 except JsonRpcError as e:
206 logger.debug("Error with node connection, e = " + repr(e)
207 + "\ntraceback = " + str(traceback.format_exc()))
208 accepting_clients = False
209 except UnknownScripthashError as e:
210 logger.debug("Disconnecting client due to misconfiguration. User"
211 + " must correctly configure master public key(s)")
212 except (IOError, EOFError) as e:
213 if isinstance(e, (EOFError, ConnectionRefusedError)):
214 logger.debug("Electrum wallet disconnected")
215 else:
216 logger.debug("IOError: " + repr(e))
217 try:
218 if sock != None:
219 sock.close()
220 except IOError:
221 pass
222 protocol.on_disconnect()
223 time.sleep(0.2)
224
225 def is_address_imported(rpc, address):
226 return rpc.call("getaddressinfo", [address])["iswatchonly"]
227
228 def get_scriptpubkeys_to_monitor(rpc, config):
229 logger = logging.getLogger('ELECTRUMPERSONALSERVER')
230 st = time.time()
231
232 deterministic_wallets = []
233 for key in config.options("master-public-keys"):
234 mpk = config.get("master-public-keys", key)
235 gaplimit = int(config.get("bitcoin-rpc", "gap_limit"))
236 chain = rpc.call("getblockchaininfo", [])["chain"]
237 try:
238 wal = deterministicwallet.parse_electrum_master_public_key(mpk,
239 gaplimit, rpc, chain)
240 except ValueError:
241 raise ValueError("Bad master public key format. Get it from " +
242 "Electrum menu `Wallet` -> `Information`")
243 deterministic_wallets.append(wal)
244 #check whether these deterministic wallets have already been imported
245 import_needed = False
246 wallets_to_import = []
247 TEST_ADDR_COUNT = 3
248 logger.info("Displaying first " + str(TEST_ADDR_COUNT) + " addresses of " +
249 "each master public key:")
250 for config_mpk_key, wal in zip(config.options("master-public-keys"),
251 deterministic_wallets):
252 first_addrs, first_spk = wal.get_addresses(change=0, from_index=0,
253 count=TEST_ADDR_COUNT)
254 logger.info("\n" + config_mpk_key + " =>\n\t" + "\n\t".join(
255 first_addrs))
256 last_addr, last_spk = wal.get_addresses(change=0, from_index=int(
257 config.get("bitcoin-rpc", "initial_import_count")) - 1, count=1)
258 if not all((is_address_imported(rpc, a) for a in (first_addrs
259 + last_addr))):
260 import_needed = True
261 wallets_to_import.append(wal)
262 logger.info("Obtaining bitcoin addresses to monitor . . .")
263 #check whether watch-only addresses have been imported
264 watch_only_addresses = []
265 for key in config.options("watch-only-addresses"):
266 watch_only_addresses.extend(config.get("watch-only-addresses",
267 key).split(' '))
268 watch_only_addresses_to_import = [a for a in watch_only_addresses
269 if not is_address_imported(rpc, a)]
270 if len(watch_only_addresses_to_import) > 0:
271 import_needed = True
272
273 if len(deterministic_wallets) == 0 and len(watch_only_addresses) == 0:
274 logger.error("No master public keys or watch-only addresses have " +
275 "been configured at all. Exiting..")
276 #import = true and none other params means exit
277 return (True, None, None)
278
279 #if addresses need to be imported then return them
280 if import_needed:
281 logger.info("Importing " + str(len(wallets_to_import))
282 + " wallets and " + str(len(watch_only_addresses_to_import))
283 + " watch-only addresses into the Bitcoin node")
284 time.sleep(5)
285 return True, watch_only_addresses_to_import, wallets_to_import
286
287 #test
288 # importing one det wallet and no addrs, two det wallets and no addrs
289 # no det wallets and some addrs, some det wallets and some addrs
290
291 #at this point we know we dont need to import any addresses
292 #find which index the deterministic wallets are up to
293 spks_to_monitor = []
294 for wal in deterministic_wallets:
295 for change in [0, 1]:
296 addrs, spks = wal.get_addresses(change, 0,
297 int(config.get("bitcoin-rpc", "initial_import_count")))
298 spks_to_monitor.extend(spks)
299 #loop until one address found that isnt imported
300 while True:
301 addrs, spks = wal.get_new_addresses(change, count=1)
302 if not is_address_imported(rpc, addrs[0]):
303 break
304 spks_to_monitor.append(spks[0])
305 wal.rewind_one(change)
306
307 spks_to_monitor.extend([hashes.address_to_script(addr, rpc)
308 for addr in watch_only_addresses])
309 et = time.time()
310 logger.info("Obtained list of addresses to monitor in " + str(et - st)
311 + "sec")
312 return False, spks_to_monitor, deterministic_wallets
313
314 def get_certs(config):
315 from pkg_resources import resource_filename
316 from electrumpersonalserver import __certfile__, __keyfile__
317
318 logger = logging.getLogger('ELECTRUMPERSONALSERVER')
319 certfile = config.get('electrum-server', 'certfile', fallback=None)
320 keyfile = config.get('electrum-server', 'keyfile', fallback=None)
321 if (certfile and keyfile) and \
322 (os.path.exists(certfile) and os.path.exists(keyfile)):
323 return certfile, keyfile
324 else:
325 certfile = resource_filename('electrumpersonalserver', __certfile__)
326 keyfile = resource_filename('electrumpersonalserver', __keyfile__)
327 if os.path.exists(certfile) and os.path.exists(keyfile):
328 return certfile, keyfile
329 else:
330 raise ValueError('invalid cert: {}, key: {}'.format(
331 certfile, keyfile))
332
333 def obtain_cookie_file_path(datadir):
334 logger = logging.getLogger('ELECTRUMPERSONALSERVER')
335 if len(datadir.strip()) == 0:
336 logger.debug("no datadir configuration, checking in default location")
337 systemname = platform.system()
338 #paths from https://en.bitcoin.it/wiki/Data_directory
339 if systemname == "Linux":
340 datadir = os.path.expanduser("~/.bitcoin")
341 elif systemname == "Windows":
342 datadir = os.path.expandvars("%APPDATA%\Bitcoin")
343 elif systemname == "Darwin": #mac os
344 datadir = os.path.expanduser(
345 "~/Library/Application Support/Bitcoin/")
346 cookie_path = os.path.join(datadir, ".cookie")
347 if not os.path.exists(cookie_path):
348 logger.warning("Unable to find .cookie file, try setting `datadir`" +
349 " config")
350 return None
351 return cookie_path
352
353 def parse_args():
354 from argparse import ArgumentParser
355
356 parser = ArgumentParser(description='Electrum Personal Server daemon')
357 parser.add_argument('config_file',
358 help='configuration file (mandatory)')
359 parser.add_argument("--rescan", action="store_true", help="Start the " +
360 " rescan script instead")
361 parser.add_argument("--rescan-date", action="store", dest="rescan_date",
362 default=None, help="Earliest wallet creation date (DD/MM/YYYY) or "
363 + "block height to rescan from")
364 parser.add_argument("-v", "--version", action="version", version=
365 "%(prog)s " + SERVER_VERSION_NUMBER)
366 return parser.parse_args()
367
368 #log for checking up/seeing your wallet, debug for when something has gone wrong
369 def logger_config(logger, config):
370 formatter = logging.Formatter(config.get("logging", "log_format",
371 fallback="%(levelname)s:%(asctime)s: %(message)s"))
372 logstream = logging.StreamHandler()
373 logstream.setFormatter(formatter)
374 logstream.setLevel(config.get("logging", "log_level_stdout", fallback=
375 "INFO"))
376 logger.addHandler(logstream)
377 filename = config.get("logging", "log_file_location", fallback="")
378 if len(filename.strip()) == 0:
379 filename= tempfile.gettempdir() + "/electrumpersonalserver.log"
380 logfile = logging.FileHandler(filename, mode=('a' if
381 config.get("logging", "append_log", fallback="false") else 'w'))
382 logfile.setFormatter(formatter)
383 logfile.setLevel(logging.DEBUG)
384 logger.addHandler(logfile)
385 logger.setLevel(logging.DEBUG)
386 return logger, filename
387
388 # returns non-zero status code on failure
389 def main():
390 opts = parse_args()
391
392 try:
393 config = RawConfigParser()
394 config.read(opts.config_file)
395 config.options("master-public-keys")
396 except NoSectionError:
397 print("ERROR: Non-existant configuration file {}".format(
398 opts.config_file))
399 return 1
400 logger = logging.getLogger('ELECTRUMPERSONALSERVER')
401 logger, logfilename = logger_config(logger, config)
402 logger.info('Starting Electrum Personal Server ' + str(
403 SERVER_VERSION_NUMBER))
404 logger.info('Logging to ' + logfilename)
405 logger.debug("Process ID (PID) = " + str(os.getpid()))
406 rpc_u = None
407 rpc_p = None
408 cookie_path = None
409 try:
410 rpc_u = config.get("bitcoin-rpc", "rpc_user")
411 rpc_p = config.get("bitcoin-rpc", "rpc_password")
412 logger.debug("obtaining auth from rpc_user/pass")
413 except NoOptionError:
414 cookie_path = obtain_cookie_file_path(config.get(
415 "bitcoin-rpc", "datadir"))
416 logger.debug("obtaining auth from .cookie")
417 if rpc_u == None and cookie_path == None:
418 return 1
419 rpc = JsonRpc(host = config.get("bitcoin-rpc", "host"),
420 port = int(config.get("bitcoin-rpc", "port")),
421 user = rpc_u, password = rpc_p, cookie_path = cookie_path,
422 wallet_filename=config.get("bitcoin-rpc", "wallet_filename").strip(),
423 logger=logger)
424
425 #TODO somewhere here loop until rpc works and fully sync'd, to allow
426 # people to run this script without waiting for their node to fully
427 # catch up sync'd when getblockchaininfo blocks == headers, or use
428 # verificationprogress
429 printed_error_msg = False
430 while bestblockhash[0] == None:
431 try:
432 bestblockhash[0] = rpc.call("getbestblockhash", [])
433 except JsonRpcError as e:
434 if not printed_error_msg:
435 logger.error("Error with bitcoin json-rpc: " + repr(e))
436 printed_error_msg = True
437 time.sleep(5)
438 try:
439 rpc.call("listunspent", [])
440 except JsonRpcError as e:
441 logger.error(repr(e))
442 logger.error("Wallet related RPC call failed, possibly the " +
443 "bitcoin node was compiled with the disable wallet flag")
444 return 1
445
446 test_keydata = (
447 "2 tpubD6NzVbkrYhZ4YVMVzC7wZeRfz3bhqcHvV8M3UiULCfzFtLtp5nwvi6LnBQegrkx" +
448 "YGPkSzXUEvcPEHcKdda8W1YShVBkhFBGkLxjSQ1Nx3cJ tpubD6NzVbkrYhZ4WjgNYq2nF" +
449 "TbiSLW2SZAzs4g5JHLqwQ3AmR3tCWpqsZJJEoZuP5HAEBNxgYQhtWMezszoaeTCg6FWGQB" +
450 "T74sszGaxaf64o5s")
451 chain = rpc.call("getblockchaininfo", [])["chain"]
452 try:
453 gaplimit = 5
454 deterministicwallet.parse_electrum_master_public_key(test_keydata,
455 gaplimit, rpc, chain)
456 except ValueError as e:
457 logger.error(repr(e))
458 logger.error("Descriptor related RPC call failed. Bitcoin Core 0.20.0"
459 + " or higher required. Exiting..")
460 return 1
461 if opts.rescan:
462 rescan_script(logger, rpc, opts.rescan_date)
463 return 0
464 while True:
465 logger.debug("Checking whether rescan is in progress")
466 walletinfo = rpc.call("getwalletinfo", [])
467 if "scanning" in walletinfo and walletinfo["scanning"]:
468 logger.debug("Waiting for Core wallet rescan to finish")
469 time.sleep(300)
470 continue
471 break
472 import_needed, relevant_spks_addrs, deterministic_wallets = \
473 get_scriptpubkeys_to_monitor(rpc, config)
474 if import_needed:
475 if not relevant_spks_addrs and not deterministic_wallets:
476 #import = true and no addresses means exit
477 return 0
478 deterministicwallet.import_addresses(rpc, relevant_spks_addrs,
479 deterministic_wallets, change_param=-1,
480 count=int(config.get("bitcoin-rpc", "initial_import_count")))
481 logger.info("Done.\nIf recovering a wallet which already has existing" +
482 " transactions, then\nrun the rescan script. If you're confident" +
483 " that the wallets are new\nand empty then there's no need to" +
484 " rescan, just restart this script")
485 else:
486 txmonitor = transactionmonitor.TransactionMonitor(rpc,
487 deterministic_wallets, logger)
488 if not txmonitor.build_address_history(relevant_spks_addrs):
489 return 1
490 try:
491 run_electrum_server(rpc, txmonitor, config)
492 except KeyboardInterrupt:
493 logger.info('Received KeyboardInterrupt, quitting')
494 return 1
495 return 0
496
497 def search_for_block_height_of_date(datestr, rpc):
498 logger = logging.getLogger('ELECTRUMPERSONALSERVER')
499 target_time = datetime.strptime(datestr, "%d/%m/%Y")
500 bestblockhash = rpc.call("getbestblockhash", [])
501 best_head = rpc.call("getblockheader", [bestblockhash])
502 if target_time > datetime.fromtimestamp(best_head["time"]):
503 logger.error("date in the future")
504 return -1
505 genesis_block = rpc.call("getblockheader", [rpc.call("getblockhash", [0])])
506 if target_time < datetime.fromtimestamp(genesis_block["time"]):
507 logger.warning("date is before the creation of bitcoin")
508 return 0
509 first_height = 0
510 last_height = best_head["height"]
511 while True:
512 m = (first_height + last_height) // 2
513 m_header = rpc.call("getblockheader", [rpc.call("getblockhash", [m])])
514 m_header_time = datetime.fromtimestamp(m_header["time"])
515 m_time_diff = (m_header_time - target_time).total_seconds()
516 if abs(m_time_diff) < 60*60*2: #2 hours
517 return m_header["height"]
518 elif m_time_diff < 0:
519 first_height = m
520 elif m_time_diff > 0:
521 last_height = m
522 else:
523 return -1
524
525 def rescan_script(logger, rpc, rescan_date):
526 if rescan_date:
527 user_input = rescan_date
528 else:
529 user_input = input("Enter earliest wallet creation date (DD/MM/YYYY) "
530 "or block height to rescan from: ")
531 try:
532 height = int(user_input)
533 except ValueError:
534 height = search_for_block_height_of_date(user_input, rpc)
535 if height == -1:
536 return
537 height -= 2016 #go back two weeks for safety
538
539 if not rescan_date:
540 if input("Rescan from block height " + str(height) + " ? (y/n):") \
541 != 'y':
542 return
543 logger.info("Rescanning. . . for progress indicator see the bitcoin node's"
544 + " debug.log file")
545 rpc.call("rescanblockchain", [height])
546 logger.info("end")
547
548 if __name__ == "__main__":
549 #entry point for pyinstaller executable
550 try:
551 res = main()
552 except:
553 res = 1
554
555 # only relevant for pyinstaller executables (on Windows):
556 if os.name == 'nt':
557 os.system("pause")
558
559 sys.exit(res)
560