homer.py 42.9 KB
Newer Older
Stephane Bortzmeyer's avatar
Stephane Bortzmeyer committed
1
2
#!/usr/bin/env python3

3
4
5
6
7
# Homer is a DoH (DNS-over-HTTPS) and DoT (DNS-over-TLS) client. Its
# main purpose is to test DoH and DoT resolvers. Reference site is
# <https://framagit.org/bortzmeyer/homer/> See author, documentation,
# etc, there, or in the README.md included with the distribution.

Stephane Bortzmeyer's avatar
Stephane Bortzmeyer committed
8
9
10
11
12
13
# http://pycurl.io/docs/latest
import pycurl

# http://www.dnspython.org/
import dns.message

14
15
16
# https://github.com/drkjam/netaddr/
import netaddr

17
18
19
20
21
# Octobre 2019: the Python GnuTLS bindings don't work with Python 3. So we use OpenSSL.
# https://www.pyopenssl.org/
# https://pyopenssl.readthedocs.io/
import OpenSSL

Stephane Bortzmeyer's avatar
Stephane Bortzmeyer committed
22
23
24
25
26
27
import io
import sys
import base64
import getopt
import urllib.parse
import time
28
29
import socket
import ctypes
30
import re
31
import os.path
32
33
import hashlib
import base64
34
import signal
Stephane Bortzmeyer's avatar
Stephane Bortzmeyer committed
35

36
# Values that can be changed from the command line
37
dot = False # DoH by default
Stephane Bortzmeyer's avatar
Stephane Bortzmeyer committed
38
verbose = False
39
debug = False
Stephane Bortzmeyer's avatar
Stephane Bortzmeyer committed
40
insecure = False
41
post = False
Stephane Bortzmeyer's avatar
Stephane Bortzmeyer committed
42
head = False
43
44
dnssec = False
edns = True
45
no_ecs = True
46
sni = True
Stephane Bortzmeyer's avatar
Stephane Bortzmeyer committed
47
rtype = 'AAAA'
48
vhostname = None
Stephane Bortzmeyer's avatar
Stephane Bortzmeyer committed
49
tests = 1 # Number of repeated tests
Alexandre's avatar
Alexandre committed
50
key = None # SPKI
51
ifile = None # Input file
52
delay = None
53
54
55
forceIPv4 = False
forceIPv6 = False
connectTo = None
56
multistreams = False
Alexandre's avatar
Alexandre committed
57
sync = False
58
display_results = True
59
show_time = False
60
check = False
61
62
mandatory_level = None
check_additional = True
63
64
65
# Monitoring plugin only:
host = None
path = None
66
expect = None
Stephane Bortzmeyer's avatar
Stephane Bortzmeyer committed
67

68
69
70
# Do not change these
re_host = re.compile(r'^([0-9a-z][0-9a-z-\.]*)|([0-9:]+)|([0-9\.])$')

71
72
73
74
75
76
77
# For the monitoring plugin
STATE_OK = 0
STATE_WARNING = 1
STATE_CRITICAL = 2
STATE_UNKNOWN = 3
STATE_DEPENDENT = 4

Alexandre's avatar
Alexandre committed
78
79
80
81
# For the check option
DOH_GET = 0
DOH_POST = 1
DOH_HEAD = 2
82
83
# Is the test mandatory?
mandatory_levels = {"legal": 30, "necessary": 20, "nicetohave": 10}
Alexandre's avatar
Alexandre committed
84

85
86
TIMEOUT_CONN = 2

87
def error(msg=None, exit=True):
Stephane Bortzmeyer's avatar
Stephane Bortzmeyer committed
88
89
    if msg is None:
        msg = "Unknown error"
90
91
    if monitoring:
        print("%s: %s" % (url, msg))
92
93
        if exit:
            sys.exit(STATE_CRITICAL)
94
    else:
Alexandre's avatar
Alexandre committed
95
96
97
        print(msg, file=sys.stderr)
        if check:
            print('KO')
98
99
        if exit:
            sys.exit(1)
Alexandre's avatar
Alexandre committed
100

Stephane Bortzmeyer's avatar
Stephane Bortzmeyer committed
101
102
103
def usage(msg=None):
    if msg:
        print(msg,file=sys.stderr)
104
105
    print("Usage: %s [--dot] url-or-servername domain-name [DNS type]" % sys.argv[0], file=sys.stderr)
    print("See the README.md for more details.", file=sys.stderr)
Stephane Bortzmeyer's avatar
Stephane Bortzmeyer committed
106

107
def is_valid_hostname(name):
108
    name = canonicalize(name)
109
    return re_host.search(name)
110

111
112
113
114
115
116
117
118
119
def canonicalize(hostname):
    result = hostname.lower()
    # TODO handle properly the case where it fails with UnicodeError
    # (two consecutive dots for instance) to get a custom exception
    result = result.encode('idna').decode()
    if result[len(result)-1] == '.':
        result = result[:-1]
    return result

120
121
122
123
def is_valid_ip_address(addr):
    try:
        baddr = netaddr.IPAddress(addr)
    except netaddr.core.AddrFormatError:
124
125
        return (False, None)
    return (True, baddr.version)
126

127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
def is_valid_url(url):
  try:
    result = urllib.parse.urlparse(url) # A very poor validation, many
    # errors (for instance whitespaces, IPv6 address litterals without
    # brackets...) are ignored.
    return (result.scheme=="https" and result.netloc != "")
  except ValueError:
    return False

def get_certificate_san(x509cert):
    san = ""
    ext_count = x509cert.get_extension_count()
    for i in range(0, ext_count):
        ext = x509cert.get_extension(i)
        if "subjectAltName" in str(ext.get_short_name()):
            san = str(ext)
    return san

145
146
147
148
149
150
151
152
153
154
155
# Try one possible name. Names must be already canonicalized.
def match_hostname(hostname, possibleMatch):
    if possibleMatch.startswith("*."): # Wildcard
        base = possibleMatch[1:] # Skip the star
        # RFC 6125 says that we MAY accept left-most labels with
        # wildcards included (foo*bar). We don't do it here.
        try:
            (first, rest) = hostname.split(".", maxsplit=1)
        except ValueError: # One-label name
            rest = hostname
        if rest == base[1:]:
156
            return True
157
        if hostname == base[1:]:
158
            return True
159
160
161
162
163
164
165
166
        return False
    else:
        return hostname == possibleMatch

# Try all the names in the certificate
def validate_hostname(hostname, cert):
    # Complete specification is in RFC 6125. It is long and
    # complicated and I'm not sure we do it perfectly.
167
    (is_addr, family) = is_valid_ip_address(hostname)
168
    hostname = canonicalize(hostname)
169
    for alt_name in get_certificate_san(cert).split(", "):
170
        if alt_name.startswith("DNS:") and not is_addr:
171
            (start, base) = alt_name.split("DNS:")
172
            base = canonicalize(base)
173
174
            found = match_hostname(hostname, base)
            if found:
175
                return True
176
177
        elif alt_name.startswith("IP Address:") and is_addr:
            host_i = netaddr.IPAddress(hostname)
178
179
180
            (start, base) = alt_name.split("IP Address:")
            if base.endswith("\n"):
                base = base[:-1]
181
182
183
184
            try:
                base_i = netaddr.IPAddress(base)
            except netaddr.core.AddrFormatError:
                continue # Ignore broken IP addresses in certificates. Are we too liberal?
Alexandre's avatar
Alexandre committed
185
            if host_i == base_i:
186
187
                return True
        else:
188
189
190
            pass # Ignore unknown alternative name types. May be
                 # accept URI alternative names for DoH,
    # According to RFC 6125, we MUST NOT try the Common Name before the Subject Alternative Names.
191
    cn = canonicalize(cert.get_subject().commonName)
192
193
194
    found = match_hostname(hostname, cn)
    if found:
        return True
195
196
    return False

197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
def check_ip_address(addr, dot=dot):
    repraddress = addr
    (is_addr, family) = is_valid_ip_address(addr)
    if not is_addr and not dot:
        raise CustomException("%s is not IPv4 and not IPv6" % addr)
    if forceIPv4 and family == 6:
        raise CustomException("You cannot force IPv4 with a litteral IPv6 address (%s)" % addr)
    elif forceIPv6 and family == 4:
        raise CustomException("You cannot force IPv6 with a litteral IPv4 address (%s)" % addr)
    if forceIPv4 or family == 4:
        family = socket.AF_INET
        repraddress = addr
    elif forceIPv6 or family == 6:
        family = socket.AF_INET6
        repraddress = f'[{addr}]'
    else:
        family = 0
    return (family, repraddress)

216
217
218
219
220
221
def dump_data(data, text="data"):
    pref = ' ' * (len(text) - 4)
    print(f'{text}: ', data)
    print(pref, 'hex:', " ".join(format(c, '02x') for c in data))
    print(pref, 'bin:', " ".join(format(c, '08b') for c in data))

222
223
224
225
226
227
def timeout_connection(signum, frame):
    raise TimeoutConnectionError('Connection timeout')

class TimeoutConnectionError(Exception):
    pass

228

Alexandre's avatar
Alexandre committed
229
230
231
232
class CustomException(Exception):
    pass


Alexandre's avatar
Alexandre committed
233
class Request:
234
    def __init__(self, qname, qtype=rtype, use_edns=edns, want_dnssec=dnssec):
235
236
237
238
239
        if no_ecs:
             opt = dns.edns.ECSOption(address='', srclen=0) # Disable ECS (RFC 7871, section 7.1.2)
             options = [opt]
        else:
            options = None
240
        self.message = dns.message.make_query(qname, dns.rdatatype.from_text(qtype),
241
                                              use_edns=use_edns, want_dnssec=want_dnssec, options=options)
242
        self.message.flags |= dns.flags.AD # Ask for validation
Alexandre's avatar
Alexandre committed
243
        self.ok = True
244
        self.i = 0 # request's number on the connection (default to the first)
Alexandre's avatar
Alexandre committed
245

246
247
248
249
    def trunc_data(self):
        self.data = self.message.to_wire()
        half = round(len(self.data) / 2)
        self.data = self.data[:half]
Alexandre's avatar
Alexandre committed
250
251

    def to_wire(self):
252
253
        self.data = self.message.to_wire()

Alexandre's avatar
Alexandre committed
254
255

class RequestDoT(Request):
Alexandre's avatar
Alexandre committed
256
    def check_response(self, debug=False):
257
        ok = self.ok
258
        if not self.rcode:
259
260
            self.ok = False
            return False
261
        if self.response.id != self.message.id:
262
            self.response = "The ID in the answer does not match the one in the query"
Alexandre's avatar
Alexandre committed
263
264
            if debug:
                self.response += f'"(query id: {self.message.id}) (response id: {self.response.id})'
265
266
267
            self.ok = False
            return False
        return self.ok
268
269


Alexandre's avatar
Alexandre committed
270
class RequestDoH(Request):
271
    def __init__(self, qname, qtype=rtype, use_edns=edns, want_dnssec=dnssec):
Alexandre's avatar
Alexandre committed
272
        Request.__init__(self, qname, qtype=rtype, use_edns=edns, want_dnssec=dnssec)
273
        self.message.id = 0 # DoH requests that
274
        self.post = False
275
        self.head = False
276

Alexandre's avatar
Alexandre committed
277
    def check_response(self, debug=False):
Alexandre's avatar
Alexandre committed
278
        ok = self.ok
279
        if self.rcode == 200:
280
281
            if self.ctype != "application/dns-message":
                self.response = "Content type of the response (\"%s\") invalid" % self.ctype
282
283
                ok = False
            else:
284
                if not self.head:
285
                    try:
Alexandre's avatar
Alexandre committed
286
                        response = dns.message.from_wire(self.response)
287
288
289
                    except dns.message.TrailingJunk: # Not DNS. Should
                        # not happen for a content type
                        # application/dns-message but who knows?
Alexandre's avatar
Alexandre committed
290
291
292
                        self.response = "ERROR Not proper DNS data, trailing junk"
                        if debug:
                            self.response += " \"%s\"" % response
293
294
                        ok = False
                    except dns.name.BadLabelType: # Not DNS.
Alexandre's avatar
Alexandre committed
295
296
297
                        self.response = "ERROR Not proper DNS data (wrong path in the URL?)"
                        if debug:
                            self.response += " \"%s\"" % response[:100]
298
                        ok = False
Alexandre's avatar
Alexandre committed
299
300
                    else:
                        self.response = response
301
                else:
302
                    if self.response_size == 0:
303
304
                        self.response = "HEAD successful"
                    else:
Alexandre's avatar
Alexandre committed
305
306
307
308
                        data = self.response
                        self.response = "ERROR Body length is not null"
                        if debug:
                            self.response += "\"%s\"" % data[:100]
309
310
311
                        ok = False
        else:
            ok = False
312
            if self.response_size == 0:
313
314
                self.response = "[No details]"
            else:
315
                self.response = self.response
Alexandre's avatar
Alexandre committed
316
        self.ok = ok
317
        return ok
318

319

320
class Connection:
321
    def __init__(self, server, servername=None, connect=None, forceIPv4=False, forceIPv6=False,
322
                 dot=dot, verbose=verbose, debug=debug, insecure=insecure):
323
        if dot and not is_valid_hostname(server):
324
            error("DoT requires a host name or IP address, not \"%s\"" % server)
325
        if not dot and not is_valid_url(server):
326
            error("DoH requires a valid HTTPS URL, not \"%s\"" % server)
327
        if forceIPv4 and forceIPv6:
Alexandre's avatar
Alexandre committed
328
            raise CustomException("Force IPv4 *or* IPv6 but not both")
329
        self.dot = dot
330
        self.server = server
331
332
        self.servername = servername
        if self.servername is not None:
Alexandre's avatar
Alexandre committed
333
            self.check = self.servername
334
        else:
Alexandre's avatar
Alexandre committed
335
            self.check = self.server
336
337
        self.dot = dot
        self.verbose = verbose
338
        self.debug = debug
339
        self.insecure = insecure
340
341
        self.forceIPv4 = forceIPv4
        self.forceIPv6 = forceIPv6
342
        self.connect_to = connect
343

344
345
    def __str__(self):
        return self.server
346

347
348
    def do_test(self, request):
        # Routine doing one actual test. Returns nothing
Alexandre's avatar
Alexandre committed
349
350
        pass

351
352
353

class ConnectionDoT(Connection):
    def __init__(self, server, servername=None, connect=None, forceIPv4=False, forceIPv6=False,
354
                 verbose=verbose, debug=debug, insecure=insecure):
355
        Connection.__init__(self, server, servername=servername, connect=connect,
356
                forceIPv4=forceIPv4, forceIPv6=forceIPv6, dot=True,
357
                verbose=verbose, debug=debug, insecure=insecure)
358
359
360
361
        if connect is not None:
            addr = connect
        else:
            addr = self.server
362
        self.family, self.repraddress = check_ip_address(self.server, dot=True)
Alexandre's avatar
Alexandre committed
363
        addrinfo_list = socket.getaddrinfo(addr, 853, self.family)
364
365
        addrinfo_set = { (addrinfo[4], addrinfo[0]) for addrinfo in addrinfo_list }
        signal.signal(signal.SIGALRM, timeout_connection)
Alexandre's avatar
Alexandre committed
366
        self.success = False
367
368
        for addrinfo in addrinfo_set:
            self.hasher = hashlib.sha256()
369
            if self.connect(addrinfo[0], addrinfo[1]):
Alexandre's avatar
Alexandre committed
370
                self.success = True
371
                break
Alexandre's avatar
Alexandre committed
372
            if self.verbose and connect is None:
Alexandre's avatar
Alexandre committed
373
                print("Trying another IP address")
Alexandre's avatar
Alexandre committed
374
375
        if not self.success:
            if self.verbose and connect is None:
Alexandre's avatar
Alexandre committed
376
                print("No other IP address")
Alexandre's avatar
Alexandre committed
377
378
379
            if connect is None:
                error(f'Could not connect to "{server}"')
            else:
380
                print(f'Could not connect to "{server}" on {connect}')
Alexandre's avatar
Alexandre committed
381

382
383
384
385
386

    def connect(self, addr, sock_family):
        signal.alarm(TIMEOUT_CONN)
        self.addr = addr
        self.sock = socket.socket(sock_family, socket.SOCK_STREAM)
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
        if self.verbose:
            print("Connecting to %s ..." % str(self.addr))
        # With typical DoT servers, we *must* use TLS 1.2 (otherwise,
        # do_handshake fails with "OpenSSL.SSL.SysCallError: (-1, 'Unexpected
        # EOF')" Typical HTTP servers are more lax.
        self.context = OpenSSL.SSL.Context(OpenSSL.SSL.TLSv1_2_METHOD)
        if self.insecure:
            self.context.set_verify(OpenSSL.SSL.VERIFY_NONE, lambda *x: True)
        else:
            self.context.set_default_verify_paths()
            self.context.set_verify_depth(4) # Seems ignored
            self.context.set_verify(OpenSSL.SSL.VERIFY_PEER | OpenSSL.SSL.VERIFY_FAIL_IF_NO_PEER_CERT | \
                                    OpenSSL.SSL.VERIFY_CLIENT_ONCE,
                                    lambda conn, cert, errno, depth, preverify_ok: preverify_ok)
        self.session = OpenSSL.SSL.Connection(self.context, self.sock)
402
403
        if sni:
            self.session.set_tlsext_host_name(canonicalize(self.check).encode()) # Server Name Indication (SNI)
404
        try:
405
            self.session.connect((self.addr))
406
            # TODO We may here have exceptions such as OpenSSL.SSL.ZeroReturnError
407
            self.session.do_handshake()
408
        except TimeoutConnectionError:
409
            if self.verbose:
Alexandre's avatar
Alexandre committed
410
                print("Timeout")
411
            return False
412
413
414
415
        except OSError:
            if self.verbose:
                print("Cannot connect")
            return False
Alexandre's avatar
Alexandre committed
416
417
418
419
        except OpenSSL.SSL.Error as e:
            if self.verbose:
                print(f"OpenSSL error: {', '.join(err[0][2] for err in e.args)}")
            return False
420
        # RFC 7858, section 4.2 and appendix A
421
        self.cert = self.session.get_peer_certificate()
422
        self.publickey = self.cert.get_pubkey()
423
        if debug or key is not None:
424
425
426
427
            self.hasher.update(OpenSSL.crypto.dump_publickey(OpenSSL.crypto.FILETYPE_ASN1,
                                                  self.publickey))
            self.digest = self.hasher.digest()
            key_string = base64.standard_b64encode(self.digest).decode()
428
        if debug:
429
430
431
432
433
            print("Certificate #%x for \"%s\", delivered by \"%s\"" % \
                  (self.cert.get_serial_number(),
                   self.cert.get_subject().commonName,
                   self.cert.get_issuer().commonName))
            print("Public key is pin-sha256=\"%s\"" % \
434
                  key_string)
435
        if not insecure:
436
437
438
439
440
441
442
            if key is None:
                valid = validate_hostname(self.check, self.cert)
                if not valid:
                    error("Certificate error: \"%s\" is not in the certificate" % (self.check))
            else:
                if key_string != key:
                    error("Key error: expected \"%s\", got \"%s\"" % (key, key_string))
443
        signal.alarm(0)
444
        return True
445

446
    def end(self):
447
448
        self.session.shutdown()
        self.session.close()
449

450
451
452
    def send_data(self, data, dump=False):
        if dump:
            dump_data(data, 'data sent')
453
454
455
        length = len(data)
        self.session.send(length.to_bytes(2, byteorder='big') + data)

456
    def receive_data(self, request, dump=False):
Alexandre's avatar
Alexandre committed
457
        buf = self.session.recv(2)
458
459
        request.response_size = int.from_bytes(buf, byteorder='big')
        buf = self.session.recv(request.response_size)
460
461
        if dump:
            dump_data(buf, 'data recv')
462
463
464
        request.response = dns.message.from_wire(buf)
        request.rcode = True

465
466
467
    def send_and_receive(self, request, dump=False):
        self.send_data(request.data, dump=dump)
        self.receive_data(request, dump=dump)
468

469
    def do_test(self, request, synchronous=True):
470
        self.send_and_receive(request)
Alexandre's avatar
Alexandre committed
471
        request.check_response(self.debug)
472
473


474
475
def create_handle(connection):
    def reset_opt_default(handle):
476
477
478
479
480
481
482
        opts = {
                pycurl.NOBODY: False,
                pycurl.POST: False,
                pycurl.POSTFIELDS: '',
                pycurl.URL: ''
               }
        for opt, value in opts.items():
483
            handle.setopt(opt, value)
484

485
    def prepare(handle, connection, request):
486
        if not connection.multistreams:
487
            handle.reset_opt_default(handle)
488
        if request.post:
489
490
491
            handle.setopt(pycurl.POST, True)
            handle.setopt(pycurl.POSTFIELDS, request.data)
            handle.setopt(pycurl.URL, connection.server)
492
        else:
493
494
495
496
497
            handle.setopt(pycurl.HTTPGET, True) # automatically sets CURLOPT_NOBODY to 0
            if request.head:
                handle.setopt(pycurl.NOBODY, True)
            dns_req = base64.urlsafe_b64encode(request.data).decode('UTF8').rstrip('=')
            handle.setopt(pycurl.URL, connection.server + ("?dns=%s" % dns_req))
498
499
500
        handle.buffer = io.BytesIO()
        handle.setopt(pycurl.WRITEDATA, handle.buffer)
        handle.request = request
501
502
503
504
505

    handle = pycurl.Curl()
    # Does not work if pycurl was not compiled with nghttp2 (recent Debian
    # packages are OK) https://github.com/pycurl/pycurl/issues/477
    handle.setopt(pycurl.HTTP_VERSION, pycurl.CURL_HTTP_VERSION_2)
Alexandre's avatar
Alexandre committed
506
    if connection.debug:
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
        handle.setopt(pycurl.VERBOSE, True)
    if connection.insecure:
        handle.setopt(pycurl.SSL_VERIFYPEER, False)
        handle.setopt(pycurl.SSL_VERIFYHOST, False)
    if connection.forceIPv4:
        handle.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_V4)
    if connection.forceIPv6:
        handle.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_V6)
    if connection.connect is not None:
        family, repraddress = check_ip_address(connection.connect, dot=False)
        handle.setopt(pycurl.CONNECT_TO, [f'::{repraddress}:443',])
    handle.setopt(pycurl.HTTPHEADER,
            ["Accept: application/dns-message", "Content-type: application/dns-message"])
    handle.reset_opt_default = reset_opt_default
    handle.prepare = prepare
    return handle
523

524
525
526

class ConnectionDoH(Connection):
    def __init__(self, server, servername=None, connect=None, forceIPv4=False, forceIPv6=False,
Alexandre's avatar
Alexandre committed
527
                 multistreams=False, verbose=verbose, debug=debug, insecure=insecure):
528
        Connection.__init__(self, server, servername=servername, connect=connect,
529
                forceIPv4=forceIPv4, forceIPv6=forceIPv6, dot=False,
Alexandre's avatar
Alexandre committed
530
                verbose=verbose, debug=debug, insecure=insecure)
531
        self.url = server
532
        self.connect = connect
Alexandre's avatar
Alexandre committed
533
        self.multistreams = multistreams
534
535
        if self.multistreams:
            self.multi = self.create_multi()
536
            self.all_handles = []
Alexandre's avatar
Alexandre committed
537
        else:
538
            self.curl_handle = create_handle(self)
539

540
541
542
543
    def create_multi(self):
        multi = pycurl.CurlMulti()
        multi.setopt(pycurl.M_MAX_HOST_CONNECTIONS, 1)
        return multi
544

545
546
547
    def end(self):
        if not self.multistreams:
            self.curl_handle.close()
548
        else:
549
            self.remove_handles()
550
            self.multi.close()
551

552
553
554
555
556
557
558
    def remove_handles(self):
        n, handle_success, handle_fail = self.multi.info_read()
        handles = handle_success + handle_fail
        for h in handles:
            h.close()
            self.multi.remove_handle(h)

559
560
561
562
563
564
565
566
567
568
569
    def perform_multi(self):
        while 1:
            ret, num_handles = self.multi.perform()
            if ret != pycurl.E_CALL_MULTI_PERFORM:
                break
        while num_handles:
            ret = self.multi.select(1.0)
            if ret == -1:
                continue
            while 1:
                ret, num_handles = self.multi.perform()
Alexandre's avatar
Alexandre committed
570
571
572
573
                if not sync:
                    n, handle_pass, handle_fail = self.multi.info_read()
                    for handle in handle_pass:
                        self.read_result_handle(handle)
574
575
                if ret != pycurl.E_CALL_MULTI_PERFORM:
                    break
576
577
578
579
        if not sync:
            n, handle_pass, handle_fail = self.multi.info_read()
            for handle in handle_pass:
                self.read_result_handle(handle)
580

581
582
583
584
585
586
587
    def send(self, handle):
        handle.buffer = io.BytesIO()
        handle.setopt(pycurl.WRITEDATA, handle.buffer)
        try:
            handle.perform()
        except pycurl.error as e:
            error(e.args[1])
Alexandre's avatar
Alexandre committed
588

589
590
591
    def receive(self, handle):
        request = handle.request
        body = handle.buffer.getvalue()
592
        body_size = len(body)
593
        http_code = handle.getinfo(pycurl.RESPONSE_CODE)
Alexandre's avatar
Alexandre committed
594
595
        handle.time = handle.getinfo(pycurl.TOTAL_TIME)
        handle.pretime = handle.getinfo(pycurl.PRETRANSFER_TIME)
596
        try:
Alexandre's avatar
Alexandre committed
597
598
            content_type = handle.getinfo(pycurl.CONTENT_TYPE)
        except TypeError: # This is the exception we get if there is no Content-Type: (for intance in response to HEAD requests)
599
            content_type = None
600
601
        request.response = body
        request.response_size = body_size
602
603
        request.rcode = http_code
        request.ctype = content_type
604
        handle.buffer.close()
605

Alexandre's avatar
Alexandre committed
606
    def send_and_receive(self, handle, dump=False):
607
608
        self.send(handle)
        self.receive(handle)
609

610
611
612
    def read_result_handle(self, handle):
        self.receive(handle)
        handle.request.check_response()
613
        if show_time:
614
            self.print_time(handle)
615
        if display_results:
616
617
618
            print("Return code %s (%.2f ms):" % (handle.request.rcode,
                (handle.time - handle.pretime) * 1000))
            print(f"{handle.request.response}\n")
Alexandre's avatar
Alexandre committed
619
620
        handle.close()
        self.multi.remove_handle(handle)
621
622
623
624
625

    def read_results(self):
        for handle in self.all_handles:
            self.read_result_handle(handle)

626
627
628
629
630
631
632
    def print_time(self, handle):
        print(f'{handle.request.i:3d}', end='   ')
        print(f'({handle.request.rcode})', end='   ')
        print(f'{handle.pretime * 1000:8.3f} ms', end='  ')
        print(f'{handle.time * 1000:8.3f} ms', end='  ')
        print(f'{(handle.time - handle.pretime) * 1000:8.3f} ms')

633
    def do_test(self, request, synchronous=True):
634
        if synchronous:
635
            handle = self.curl_handle
636
        else:
637
            handle = create_handle(self)
638
            self.all_handles.append(handle)
639
        handle.prepare(handle, self, request)
Alexandre's avatar
Alexandre committed
640
        if synchronous:
641
            self.send_and_receive(handle)
Alexandre's avatar
Alexandre committed
642
            request.check_response(self.debug)
643
        else:
644
            self.multi.add_handle(handle)
645
646
647
648
649
650
651
652
653
654
655
656
657


def get_next_domain(input_file):
    name, rtype = 'framagit.org', 'AAAA'
    line = input_file.readline()
    if line[:-1] == "":
        error("Not enough data in %s for the %i tests" % (ifile, tests))
    if line.find(' ') == -1:
        name = line[:-1]
        rtype = 'AAAA'
    else:
        (name, rtype) = line.split()
    return name, rtype
Alexandre's avatar
Alexandre committed
658

659
def print_result(connection, request, prefix=None, display_err=True):
Alexandre's avatar
Alexandre committed
660
    ok = request.ok
661
662
663
664
665
666
667
    dot = connection.dot
    server = connection.server
    rcode = request.rcode
    msg = request.response
    size = request.response_size
    if (dot and rcode) or (not dot and rcode == 200):
        if not monitoring:
668
669
            if not dot and show_time:
                connection.print_time(connection.curl_handle)
670
            if display_results and (not check or verbose):
671
                print(msg)
672
        else:
673
            if expect is not None and expect not in str(request.response):
674
                ok = False
675
676
                print("%s Cannot find \"%s\" in response" % (server, expect))
                sys.exit(STATE_CRITICAL)
677
            if ok and size is not None and size > 0:
678
                print("%s OK - %s" % (server, "No error for %s/%s, %i bytes received" % (name, rtype, size)))
679
            else:
680
681
682
683
                print("%s OK - %s" % (server, "No error"))
            sys.exit(STATE_OK)
    else:
        if not monitoring:
684
            if display_err:
685
686
                if check:
                    print(connection.connect_to, end=': ', file=sys.stderr)
687
688
689
690
691
692
693
694
695
696
                if prefix:
                    print(prefix, end=': ', file=sys.stderr)
                if dot:
                    print("Error: %s" % msg, file=sys.stderr)
                else:
                   try:
                       msg = msg.decode()
                   except (UnicodeDecodeError, AttributeError):
                       pass # Sometimes, msg can be binary, or Latin-1
                   print("HTTP error %i: %s" % (rcode, msg), file=sys.stderr) 
697
        else:
698
699
            if not dot:
                print("%s HTTP error - %i: %s" % (server, rcode, msg))
700
            else:
701
702
703
704
705
                print("%s Error - %i: %s" % (server, rcode, msg))
            sys.exit(STATE_CRITICAL)
        ok = False
    return ok

706
def create_request(qname, qtype=rtype, use_edns=edns, want_dnssec=dnssec, dot=dot, trunc=False):
Alexandre's avatar
Alexandre committed
707
    if dot:
708
        request = RequestDoT(qname, rtype, use_edns, want_dnssec)
Alexandre's avatar
Alexandre committed
709
    else:
710
        request = RequestDoH(qname, rtype, use_edns, want_dnssec)
Alexandre's avatar
Alexandre committed
711
712
713
714
715
716
717
718
719
    if trunc:
        request.trunc_data()
    else:
        request.to_wire()
    return request

def create_requests_list(dot=dot, **req_args):
    requests = []
    if dot:
720
721
722
723
724
725
        requests.append(('Test 1', create_request(dot=dot, **req_args),
                        mandatory_levels["legal"]))
        requests.append(('Test 2', create_request(dot=dot, **req_args),
                         mandatory_levels["necessary"])) # RFC 7858,
        # section 3.3, SHOULD accept several requests on one connection.
        # TODO we miss the tests of pipelining and out-of-order.
Alexandre's avatar
Alexandre committed
726
    else:
727
728
729
730
731
732
733
734
        requests.append(('Test GET', create_request(**req_args), DOH_GET,
                         mandatory_levels["legal"])) # RFC 8484, section 4.1
        requests.append(('Test POST', create_request(**req_args), DOH_POST,
                         mandatory_levels["legal"])) # RFC 8484, section 4.1
        requests.append(('Test HEAD', create_request(**req_args), DOH_HEAD,
                         mandatory_levels["nicetohave"])) # HEAD
        # method is not mentioned in RFC 8484 (see section 4.1), so
        # just "nice to have".
Alexandre's avatar
Alexandre committed
735
    return requests
736

737
def run_check_default(connection):
Alexandre's avatar
Alexandre committed
738
    ok = True
739
740
741
742
    req_args = { 'qname': name, 'qtype': rtype, 'use_edns': edns, 'want_dnssec': dnssec }
    requests = create_requests_list(dot=dot, **req_args)
    for request_pack in requests:
        if dot:
743
            test_name, request, mandatory = request_pack
744
        else:
745
            test_name, request, method, mandatory = request_pack
746
747
        if verbose:
            print(test_name)
Alexandre's avatar
Alexandre committed
748
749
750
        if dot:
            bundle = request
        else:
751
752
753
754
            if method == DOH_POST:
                request.post = True
            elif method == DOH_HEAD:
                request.head = True
755
756
757
            handle = connection.curl_handle
            handle.prepare(handle, connection, request)
            bundle = handle
758
        try:
759
            connection.send_and_receive(bundle)
760
761
762
763
        except CustomException as e:
            ok = False
            error(e)
            break
Alexandre's avatar
Alexandre committed
764
        request.check_response(debug)
765
766
767
768
        if not print_result(connection, request, prefix=test_name, display_err=False):
            if mandatory >= mandatory_level:
                print_result(connection, request, prefix=test_name, display_err=True)
                ok = False
Alexandre's avatar
Alexandre committed
769
770
            if verbose:
                print()
771
            break
Alexandre's avatar
Alexandre committed
772
773
        if verbose:
            print()
Alexandre's avatar
Alexandre committed
774
    return ok
775

776
777
778
779
780
def run_check_mime(connection, accept="application/dns-message", content_type="application/dns-message"):
    if dot:
        return True
    ok = True
    header = [f"Accept: {accept}", f"Content-type: {content_type}"]
Alexandre's avatar
Alexandre committed
781
782
783
    if verbose:
        test_name = f'Test mime: {", ".join(h for h in header)}'
        print(test_name)
784
785
    req_args = { 'qname': name, 'qtype': rtype, 'use_edns': edns, 'want_dnssec': dnssec }
    request = create_request(**req_args)
786
787
788
    handle = connection.curl_handle
    handle.setopt(pycurl.HTTPHEADER, header)
    handle.prepare(handle, connection, request)
789
    try:
790
        connection.send_and_receive(handle)
791
792
793
    except CustomException as e:
        ok = False
        error(e)
Alexandre's avatar
Alexandre committed
794
    request.check_response(debug)
795
796
797
798
    if not print_result(connection, request, prefix=f"Test Header {', '.join(header)}"):
        ok = False
    default = "application/dns-message"
    default_header = [f"Accept: {default}", f"Content-type: {default}"]
799
    handle.setopt(pycurl.HTTPHEADER, default_header)
Alexandre's avatar
Alexandre committed
800
801
    if verbose:
        print()
802
803
    return ok

804
805
806
807
808
809
810
811
def run_check_trunc(connection):
    ok = True
    test_name = 'Test truncated data'
    if verbose:
        print(test_name)
    req_args = { 'qname': name, 'qtype': rtype, 'use_edns': edns, 'want_dnssec': dnssec }
    if dot:
        request = create_request(dot=dot, trunc=True, **req_args)
812
        bundle = request
813
814
815
    else:
        request = create_request(trunc=True, **req_args)
        request.post = True
816
817
818
        handle = connection.curl_handle
        handle.prepare(handle, connection, request)
        bundle = handle
819
    try:
820
        # 8.8.8.8 replies FORMERR but most DoT servers violently shut down the connection (which is legal)
Alexandre's avatar
Alexandre committed
821
        connection.send_and_receive(bundle, dump=debug)
822
823
824
    except CustomException as e:
        ok = False
        error(e)
825
826
    except OpenSSL.SSL.ZeroReturnError: # This is acceptable
        return ok
827
828
829
830
831
    except dns.exception.FormError: # This is also acceptable
        # Some DSN resolvers will echo mangled requests with
        # the RCODE set to FORMERR
        # so response can not be parsed in this case
        return ok
Alexandre's avatar
Alexandre committed
832
    if request.check_response(debug): # FORMERR is expected
Alexandre's avatar
Alexandre committed
833
834
835
836
        if dot:
            ok = request.rcode == dns.rcode.FORMERR
        else:
            ok = (request.response.rcode() == dns.rcode.FORMERR)
837
    else:
Alexandre's avatar
Alexandre committed
838
839
840
841
842
        if dot:
            ok = False
        else: # a 400 response's status is acceptable
            ok = (request.rcode >= 400 and request.rcode < 500)
    print_result(connection, request, prefix=test_name, display_err=not ok)
Alexandre's avatar
Alexandre committed
843
844
    if verbose:
        print()
845
846
    return ok

847
def run_check_additionals(connection):
848
849
    if not run_check_trunc(connection):
        return False
850
851
852
853
854
    # The DoH server is right to reject these (Example: 'HTTP
    # error 415: only Content-Type: application/dns-message is
    # supported')
    run_check_mime(connection, accept="text/html")
    run_check_mime(connection, content_type="text/html")
855
856
857
858
859
    return True

def run_check(connection):
    if not run_check_default(connection):
        return False
860
    if check_additional and not run_check_additionals(connection):
861
862
863
        return False
    return True

864
865
866
867
868
869
870
871
def resolved_ips(host, port, family, dot=dot):
    try:
        addr_list = socket.getaddrinfo(host, port, family)
    except socket.gaierror:
        error(f'Could not resolve "{url}"')
    ip_set = { addr[4][0] for addr in addr_list }
    return ip_set

872
# Main program
873
874
875
876
877
878
me = os.path.basename(sys.argv[0])
monitoring = (me == "check_doh" or me == "check_dot")
if not monitoring:
    name = None
    message = None
    try:
879
        optlist, args = getopt.getopt (sys.argv[1:], "hvPkeV:r:f:d:t46",
880
                                       ["help", "verbose", "debug", "dot", "head",
Alexandre's avatar
Alexandre committed
881
                                        "insecure", "POST", "vhost=", "multistreams",
882
                                        "sync", "no-display-results", "time",
883
                                        "dnssec", "noedns", "ecs", "repeat=", "file=", "delay=",
884
                                        "key=", "nosni",
885
                                        "v4only", "v6only",
886
                                        "check", "mandatory-level="])
887
888
889
890
        for option, value in optlist:
            if option == "--help" or option == "-h":
                usage()
                sys.exit(0)
891
892
            elif option == "--dot" or option == "-t":
                dot = True
893
894
            elif option == "--verbose" or option == "-v":
                verbose = True
Alexandre's avatar
Alexandre committed
895
            elif option == "--debug":
896
897
                debug = True
                verbose = True
898
            elif option == "--HEAD" or option == "--head" or option == "-e":
899
                head = True
900
            elif option == "--POST" or option == "--post" or option == "-P":
901
                post = True
902
903
            elif option == "--vhost" or option == "-V":
                vhostname = value
904
905
            elif option == "--insecure" or option == "-k":
                insecure = True
906
907
            elif option == "--multistreams":
                multistreams = True
Alexandre's avatar
Alexandre committed
908
909
            elif option == "--sync":
                sync = True
910
911
            elif option == "--no-display-results":
                display_results = False
912
913
            elif option == "--time":
                show_time = True