homer.py 42.7 KB
Newer Older
Stephane Bortzmeyer's avatar
Stephane Bortzmeyer committed
1
2
#!/usr/bin/env python3

3
4
5
6
7
# Homer is a DoH (DNS-over-HTTPS) and DoT (DNS-over-TLS) client. Its
# main purpose is to test DoH and DoT resolvers. Reference site is
# <https://framagit.org/bortzmeyer/homer/> See author, documentation,
# etc, there, or in the README.md included with the distribution.

Stephane Bortzmeyer's avatar
Stephane Bortzmeyer committed
8
9
10
11
12
13
# http://pycurl.io/docs/latest
import pycurl

# http://www.dnspython.org/
import dns.message

14
15
16
# https://github.com/drkjam/netaddr/
import netaddr

17
18
19
20
21
# Octobre 2019: the Python GnuTLS bindings don't work with Python 3. So we use OpenSSL.
# https://www.pyopenssl.org/
# https://pyopenssl.readthedocs.io/
import OpenSSL

Stephane Bortzmeyer's avatar
Stephane Bortzmeyer committed
22
23
24
25
26
27
import io
import sys
import base64
import getopt
import urllib.parse
import time
28
29
import socket
import ctypes
30
import re
31
import os.path
32
33
import hashlib
import base64
34
import signal
Stephane Bortzmeyer's avatar
Stephane Bortzmeyer committed
35

36
# Values that can be changed from the command line
37
dot = False # DoH by default
Stephane Bortzmeyer's avatar
Stephane Bortzmeyer committed
38
verbose = False
39
debug = False
Stephane Bortzmeyer's avatar
Stephane Bortzmeyer committed
40
insecure = False
41
post = False
Stephane Bortzmeyer's avatar
Stephane Bortzmeyer committed
42
head = False
43
44
dnssec = False
edns = True
45
no_ecs = True
46
sni = True
Stephane Bortzmeyer's avatar
Stephane Bortzmeyer committed
47
rtype = 'AAAA'
48
vhostname = None
Stephane Bortzmeyer's avatar
Stephane Bortzmeyer committed
49
tests = 1 # Number of repeated tests
Alexandre's avatar
Alexandre committed
50
key = None # SPKI
51
ifile = None # Input file
52
delay = None
53
54
55
forceIPv4 = False
forceIPv6 = False
connectTo = None
56
multistreams = False
Alexandre's avatar
Alexandre committed
57
sync = False
58
display_results = True
59
show_time = False
60
check = False
61
62
mandatory_level = None
check_additional = True
63
64
65
# Monitoring plugin only:
host = None
path = None
66
expect = None
Stephane Bortzmeyer's avatar
Stephane Bortzmeyer committed
67

68
69
70
# Do not change these
re_host = re.compile(r'^([0-9a-z][0-9a-z-\.]*)|([0-9:]+)|([0-9\.])$')

71
72
73
74
75
76
77
# For the monitoring plugin
STATE_OK = 0
STATE_WARNING = 1
STATE_CRITICAL = 2
STATE_UNKNOWN = 3
STATE_DEPENDENT = 4

Alexandre's avatar
Alexandre committed
78
79
80
81
# For the check option
DOH_GET = 0
DOH_POST = 1
DOH_HEAD = 2
82
83
# Is the test mandatory?
mandatory_levels = {"legal": 30, "necessary": 20, "nicetohave": 10}
Alexandre's avatar
Alexandre committed
84

85
86
TIMEOUT_CONN = 2

87
def error(msg=None, exit=True):
Stephane Bortzmeyer's avatar
Stephane Bortzmeyer committed
88
89
    if msg is None:
        msg = "Unknown error"
90
91
    if monitoring:
        print("%s: %s" % (url, msg))
92
93
        if exit:
            sys.exit(STATE_CRITICAL)
94
    else:
Alexandre's avatar
Alexandre committed
95
96
97
        print(msg, file=sys.stderr)
        if check:
            print('KO')
98
99
        if exit:
            sys.exit(1)
Alexandre's avatar
Alexandre committed
100

Stephane Bortzmeyer's avatar
Stephane Bortzmeyer committed
101
102
103
def usage(msg=None):
    if msg:
        print(msg,file=sys.stderr)
104
105
    print("Usage: %s [--dot] url-or-servername domain-name [DNS type]" % sys.argv[0], file=sys.stderr)
    print("See the README.md for more details.", file=sys.stderr)
Stephane Bortzmeyer's avatar
Stephane Bortzmeyer committed
106

107
def is_valid_hostname(name):
108
    name = canonicalize(name)
109
    return re_host.search(name)
110

111
112
113
114
115
116
117
118
119
def canonicalize(hostname):
    result = hostname.lower()
    # TODO handle properly the case where it fails with UnicodeError
    # (two consecutive dots for instance) to get a custom exception
    result = result.encode('idna').decode()
    if result[len(result)-1] == '.':
        result = result[:-1]
    return result

120
121
122
123
def is_valid_ip_address(addr):
    try:
        baddr = netaddr.IPAddress(addr)
    except netaddr.core.AddrFormatError:
124
125
        return (False, None)
    return (True, baddr.version)
126

127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
def is_valid_url(url):
  try:
    result = urllib.parse.urlparse(url) # A very poor validation, many
    # errors (for instance whitespaces, IPv6 address litterals without
    # brackets...) are ignored.
    return (result.scheme=="https" and result.netloc != "")
  except ValueError:
    return False

def get_certificate_san(x509cert):
    san = ""
    ext_count = x509cert.get_extension_count()
    for i in range(0, ext_count):
        ext = x509cert.get_extension(i)
        if "subjectAltName" in str(ext.get_short_name()):
            san = str(ext)
    return san

145
146
147
148
149
150
151
152
153
154
155
# Try one possible name. Names must be already canonicalized.
def match_hostname(hostname, possibleMatch):
    if possibleMatch.startswith("*."): # Wildcard
        base = possibleMatch[1:] # Skip the star
        # RFC 6125 says that we MAY accept left-most labels with
        # wildcards included (foo*bar). We don't do it here.
        try:
            (first, rest) = hostname.split(".", maxsplit=1)
        except ValueError: # One-label name
            rest = hostname
        if rest == base[1:]:
156
            return True
157
        if hostname == base[1:]:
158
            return True
159
160
161
162
163
164
165
166
        return False
    else:
        return hostname == possibleMatch

# Try all the names in the certificate
def validate_hostname(hostname, cert):
    # Complete specification is in RFC 6125. It is long and
    # complicated and I'm not sure we do it perfectly.
167
    (is_addr, family) = is_valid_ip_address(hostname)
168
    hostname = canonicalize(hostname)
169
    for alt_name in get_certificate_san(cert).split(", "):
170
        if alt_name.startswith("DNS:") and not is_addr:
171
            (start, base) = alt_name.split("DNS:")
172
            base = canonicalize(base)
173
174
            found = match_hostname(hostname, base)
            if found:
175
                return True
176
177
        elif alt_name.startswith("IP Address:") and is_addr:
            host_i = netaddr.IPAddress(hostname)
178
179
180
            (start, base) = alt_name.split("IP Address:")
            if base.endswith("\n"):
                base = base[:-1]
181
182
183
184
            try:
                base_i = netaddr.IPAddress(base)
            except netaddr.core.AddrFormatError:
                continue # Ignore broken IP addresses in certificates. Are we too liberal?
Alexandre's avatar
Alexandre committed
185
            if host_i == base_i:
186
187
                return True
        else:
188
189
190
            pass # Ignore unknown alternative name types. May be
                 # accept URI alternative names for DoH,
    # According to RFC 6125, we MUST NOT try the Common Name before the Subject Alternative Names.
191
    cn = canonicalize(cert.get_subject().commonName)
192
193
194
    found = match_hostname(hostname, cn)
    if found:
        return True
195
196
    return False

197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
def check_ip_address(addr, dot=dot):
    repraddress = addr
    (is_addr, family) = is_valid_ip_address(addr)
    if not is_addr and not dot:
        raise CustomException("%s is not IPv4 and not IPv6" % addr)
    if forceIPv4 and family == 6:
        raise CustomException("You cannot force IPv4 with a litteral IPv6 address (%s)" % addr)
    elif forceIPv6 and family == 4:
        raise CustomException("You cannot force IPv6 with a litteral IPv4 address (%s)" % addr)
    if forceIPv4 or family == 4:
        family = socket.AF_INET
        repraddress = addr
    elif forceIPv6 or family == 6:
        family = socket.AF_INET6
        repraddress = f'[{addr}]'
    else:
        family = 0
    return (family, repraddress)

216
217
218
219
220
221
def dump_data(data, text="data"):
    pref = ' ' * (len(text) - 4)
    print(f'{text}: ', data)
    print(pref, 'hex:', " ".join(format(c, '02x') for c in data))
    print(pref, 'bin:', " ".join(format(c, '08b') for c in data))

222
223
224
225
226
227
def timeout_connection(signum, frame):
    raise TimeoutConnectionError('Connection timeout')

class TimeoutConnectionError(Exception):
    pass

228

Alexandre's avatar
Alexandre committed
229
230
231
232
class CustomException(Exception):
    pass


Alexandre's avatar
Alexandre committed
233
class Request:
234
    def __init__(self, qname, qtype=rtype, use_edns=edns, want_dnssec=dnssec):
235
236
237
238
239
        if no_ecs:
             opt = dns.edns.ECSOption(address='', srclen=0) # Disable ECS (RFC 7871, section 7.1.2)
             options = [opt]
        else:
            options = None
240
        self.message = dns.message.make_query(qname, dns.rdatatype.from_text(qtype),
241
                                              use_edns=use_edns, want_dnssec=want_dnssec, options=options)
242
        self.message.flags |= dns.flags.AD # Ask for validation
Alexandre's avatar
Alexandre committed
243
        self.ok = True
Alexandre's avatar
Alexandre committed
244

245
246
247
248
    def trunc_data(self):
        self.data = self.message.to_wire()
        half = round(len(self.data) / 2)
        self.data = self.data[:half]
Alexandre's avatar
Alexandre committed
249
250

    def to_wire(self):
251
252
        self.data = self.message.to_wire()

Alexandre's avatar
Alexandre committed
253
254

class RequestDoT(Request):
Alexandre's avatar
Alexandre committed
255
    def check_response(self, debug=False):
256
        ok = self.ok
257
        if not self.rcode:
258
259
            self.ok = False
            return False
260
        if self.response.id != self.message.id:
261
            self.response = "The ID in the answer does not match the one in the query"
Alexandre's avatar
Alexandre committed
262
263
            if debug:
                self.response += f'"(query id: {self.message.id}) (response id: {self.response.id})'
264
265
266
            self.ok = False
            return False
        return self.ok
267
268


Alexandre's avatar
Alexandre committed
269
class RequestDoH(Request):
270
    def __init__(self, qname, qtype=rtype, use_edns=edns, want_dnssec=dnssec):
Alexandre's avatar
Alexandre committed
271
        Request.__init__(self, qname, qtype=rtype, use_edns=edns, want_dnssec=dnssec)
272
        self.message.id = 0 # DoH requests that
273
        self.post = False
274
        self.head = False
275

Alexandre's avatar
Alexandre committed
276
    def check_response(self, debug=False):
Alexandre's avatar
Alexandre committed
277
        ok = self.ok
278
        if self.rcode == 200:
279
280
            if self.ctype != "application/dns-message":
                self.response = "Content type of the response (\"%s\") invalid" % self.ctype
281
282
                ok = False
            else:
283
                if not self.head:
284
                    try:
Alexandre's avatar
Alexandre committed
285
                        response = dns.message.from_wire(self.response)
286
287
288
                    except dns.message.TrailingJunk: # Not DNS. Should
                        # not happen for a content type
                        # application/dns-message but who knows?
Alexandre's avatar
Alexandre committed
289
290
291
                        self.response = "ERROR Not proper DNS data, trailing junk"
                        if debug:
                            self.response += " \"%s\"" % response
292
293
                        ok = False
                    except dns.name.BadLabelType: # Not DNS.
Alexandre's avatar
Alexandre committed
294
295
296
                        self.response = "ERROR Not proper DNS data (wrong path in the URL?)"
                        if debug:
                            self.response += " \"%s\"" % response[:100]
297
                        ok = False
Alexandre's avatar
Alexandre committed
298
299
                    else:
                        self.response = response
300
                else:
301
                    if self.response_size == 0:
302
303
                        self.response = "HEAD successful"
                    else:
Alexandre's avatar
Alexandre committed
304
305
306
307
                        data = self.response
                        self.response = "ERROR Body length is not null"
                        if debug:
                            self.response += "\"%s\"" % data[:100]
308
309
310
                        ok = False
        else:
            ok = False
311
            if self.response_size == 0:
312
313
                self.response = "[No details]"
            else:
314
                self.response = self.response
Alexandre's avatar
Alexandre committed
315
        self.ok = ok
316
        return ok
317

318

319
class Connection:
320
    def __init__(self, server, servername=None, connect=None, forceIPv4=False, forceIPv6=False,
321
                 dot=dot, verbose=verbose, debug=debug, insecure=insecure):
322
        if dot and not is_valid_hostname(server):
323
            error("DoT requires a host name or IP address, not \"%s\"" % server)
324
        if not dot and not is_valid_url(server):
325
            error("DoH requires a valid HTTPS URL, not \"%s\"" % server)
326
        if forceIPv4 and forceIPv6:
Alexandre's avatar
Alexandre committed
327
            raise CustomException("Force IPv4 *or* IPv6 but not both")
328
        self.dot = dot
329
        self.server = server
330
331
        self.servername = servername
        if self.servername is not None:
Alexandre's avatar
Alexandre committed
332
            self.check = self.servername
333
        else:
Alexandre's avatar
Alexandre committed
334
            self.check = self.server
335
336
        self.dot = dot
        self.verbose = verbose
337
        self.debug = debug
338
        self.insecure = insecure
339
340
        self.forceIPv4 = forceIPv4
        self.forceIPv6 = forceIPv6
341
        self.connect_to = connect
342

343
344
    def __str__(self):
        return self.server
345

346
347
    def do_test(self, request):
        # Routine doing one actual test. Returns nothing
Alexandre's avatar
Alexandre committed
348
349
        pass

350
351
352

class ConnectionDoT(Connection):
    def __init__(self, server, servername=None, connect=None, forceIPv4=False, forceIPv6=False,
353
                 verbose=verbose, debug=debug, insecure=insecure):
354
        Connection.__init__(self, server, servername=servername, connect=connect,
355
                forceIPv4=forceIPv4, forceIPv6=forceIPv6, dot=True,
356
                verbose=verbose, debug=debug, insecure=insecure)
357
358
359
360
        if connect is not None:
            addr = connect
        else:
            addr = self.server
361
        self.family, self.repraddress = check_ip_address(self.server, dot=True)
Alexandre's avatar
Alexandre committed
362
        addrinfo_list = socket.getaddrinfo(addr, 853, self.family)
363
364
        addrinfo_set = { (addrinfo[4], addrinfo[0]) for addrinfo in addrinfo_list }
        signal.signal(signal.SIGALRM, timeout_connection)
Alexandre's avatar
Alexandre committed
365
        self.success = False
366
367
        for addrinfo in addrinfo_set:
            self.hasher = hashlib.sha256()
368
            if self.connect(addrinfo[0], addrinfo[1]):
Alexandre's avatar
Alexandre committed
369
                self.success = True
370
                break
Alexandre's avatar
Alexandre committed
371
            if self.verbose and connect is None:
Alexandre's avatar
Alexandre committed
372
                print("Trying another IP address")
Alexandre's avatar
Alexandre committed
373
374
        if not self.success:
            if self.verbose and connect is None:
Alexandre's avatar
Alexandre committed
375
                print("No other IP address")
Alexandre's avatar
Alexandre committed
376
377
378
            if connect is None:
                error(f'Could not connect to "{server}"')
            else:
379
                print(f'Could not connect to "{server}" on {connect}')
Alexandre's avatar
Alexandre committed
380

381
382
383
384
385

    def connect(self, addr, sock_family):
        signal.alarm(TIMEOUT_CONN)
        self.addr = addr
        self.sock = socket.socket(sock_family, socket.SOCK_STREAM)
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
        if self.verbose:
            print("Connecting to %s ..." % str(self.addr))
        # With typical DoT servers, we *must* use TLS 1.2 (otherwise,
        # do_handshake fails with "OpenSSL.SSL.SysCallError: (-1, 'Unexpected
        # EOF')" Typical HTTP servers are more lax.
        self.context = OpenSSL.SSL.Context(OpenSSL.SSL.TLSv1_2_METHOD)
        if self.insecure:
            self.context.set_verify(OpenSSL.SSL.VERIFY_NONE, lambda *x: True)
        else:
            self.context.set_default_verify_paths()
            self.context.set_verify_depth(4) # Seems ignored
            self.context.set_verify(OpenSSL.SSL.VERIFY_PEER | OpenSSL.SSL.VERIFY_FAIL_IF_NO_PEER_CERT | \
                                    OpenSSL.SSL.VERIFY_CLIENT_ONCE,
                                    lambda conn, cert, errno, depth, preverify_ok: preverify_ok)
        self.session = OpenSSL.SSL.Connection(self.context, self.sock)
401
402
        if sni:
            self.session.set_tlsext_host_name(canonicalize(self.check).encode()) # Server Name Indication (SNI)
403
        try:
404
            self.session.connect((self.addr))
405
            # TODO We may here have exceptions such as OpenSSL.SSL.ZeroReturnError
406
            self.session.do_handshake()
407
        except TimeoutConnectionError:
408
            if self.verbose:
Alexandre's avatar
Alexandre committed
409
                print("Timeout")
410
            return False
411
412
413
414
        except OSError:
            if self.verbose:
                print("Cannot connect")
            return False
Alexandre's avatar
Alexandre committed
415
416
417
418
        except OpenSSL.SSL.Error as e:
            if self.verbose:
                print(f"OpenSSL error: {', '.join(err[0][2] for err in e.args)}")
            return False
419
        # RFC 7858, section 4.2 and appendix A
420
        self.cert = self.session.get_peer_certificate()
421
        self.publickey = self.cert.get_pubkey()
422
        if debug or key is not None:
423
424
425
426
            self.hasher.update(OpenSSL.crypto.dump_publickey(OpenSSL.crypto.FILETYPE_ASN1,
                                                  self.publickey))
            self.digest = self.hasher.digest()
            key_string = base64.standard_b64encode(self.digest).decode()
427
        if debug:
428
429
430
431
432
            print("Certificate #%x for \"%s\", delivered by \"%s\"" % \
                  (self.cert.get_serial_number(),
                   self.cert.get_subject().commonName,
                   self.cert.get_issuer().commonName))
            print("Public key is pin-sha256=\"%s\"" % \
433
                  key_string)
434
        if not insecure:
435
436
437
438
439
440
441
            if key is None:
                valid = validate_hostname(self.check, self.cert)
                if not valid:
                    error("Certificate error: \"%s\" is not in the certificate" % (self.check))
            else:
                if key_string != key:
                    error("Key error: expected \"%s\", got \"%s\"" % (key, key_string))
442
        signal.alarm(0)
443
        return True
444

445
    def end(self):
446
447
        self.session.shutdown()
        self.session.close()
448

449
450
451
    def send_data(self, data, dump=False):
        if dump:
            dump_data(data, 'data sent')
452
453
454
        length = len(data)
        self.session.send(length.to_bytes(2, byteorder='big') + data)

455
    def receive_data(self, request, dump=False):
Alexandre's avatar
Alexandre committed
456
        buf = self.session.recv(2)
457
458
        request.response_size = int.from_bytes(buf, byteorder='big')
        buf = self.session.recv(request.response_size)
459
460
        if dump:
            dump_data(buf, 'data recv')
461
462
463
        request.response = dns.message.from_wire(buf)
        request.rcode = True

464
465
466
    def send_and_receive(self, request, dump=False):
        self.send_data(request.data, dump=dump)
        self.receive_data(request, dump=dump)
467

468
    def do_test(self, request, synchronous=True):
469
        self.send_and_receive(request)
Alexandre's avatar
Alexandre committed
470
        request.check_response(self.debug)
471
472


473
474
def create_handle(connection):
    def reset_opt_default(handle):
475
476
477
478
479
480
481
        opts = {
                pycurl.NOBODY: False,
                pycurl.POST: False,
                pycurl.POSTFIELDS: '',
                pycurl.URL: ''
               }
        for opt, value in opts.items():
482
            handle.setopt(opt, value)
483

484
    def prepare(handle, connection, request):
485
        if not connection.multistreams:
486
            handle.reset_opt_default(handle)
487
        if request.post:
488
489
490
            handle.setopt(pycurl.POST, True)
            handle.setopt(pycurl.POSTFIELDS, request.data)
            handle.setopt(pycurl.URL, connection.server)
491
        else:
492
493
494
495
496
            handle.setopt(pycurl.HTTPGET, True) # automatically sets CURLOPT_NOBODY to 0
            if request.head:
                handle.setopt(pycurl.NOBODY, True)
            dns_req = base64.urlsafe_b64encode(request.data).decode('UTF8').rstrip('=')
            handle.setopt(pycurl.URL, connection.server + ("?dns=%s" % dns_req))
497
498
499
        handle.buffer = io.BytesIO()
        handle.setopt(pycurl.WRITEDATA, handle.buffer)
        handle.request = request
500
501
502
503
504

    handle = pycurl.Curl()
    # Does not work if pycurl was not compiled with nghttp2 (recent Debian
    # packages are OK) https://github.com/pycurl/pycurl/issues/477
    handle.setopt(pycurl.HTTP_VERSION, pycurl.CURL_HTTP_VERSION_2)
Alexandre's avatar
Alexandre committed
505
    if connection.debug:
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
        handle.setopt(pycurl.VERBOSE, True)
    if connection.insecure:
        handle.setopt(pycurl.SSL_VERIFYPEER, False)
        handle.setopt(pycurl.SSL_VERIFYHOST, False)
    if connection.forceIPv4:
        handle.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_V4)
    if connection.forceIPv6:
        handle.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_V6)
    if connection.connect is not None:
        family, repraddress = check_ip_address(connection.connect, dot=False)
        handle.setopt(pycurl.CONNECT_TO, [f'::{repraddress}:443',])
    handle.setopt(pycurl.HTTPHEADER,
            ["Accept: application/dns-message", "Content-type: application/dns-message"])
    handle.reset_opt_default = reset_opt_default
    handle.prepare = prepare
    return handle
522

523
524
525

class ConnectionDoH(Connection):
    def __init__(self, server, servername=None, connect=None, forceIPv4=False, forceIPv6=False,
Alexandre's avatar
Alexandre committed
526
                 multistreams=False, verbose=verbose, debug=debug, insecure=insecure):
527
        Connection.__init__(self, server, servername=servername, connect=connect,
528
                forceIPv4=forceIPv4, forceIPv6=forceIPv6, dot=False,
Alexandre's avatar
Alexandre committed
529
                verbose=verbose, debug=debug, insecure=insecure)
530
        self.url = server
531
        self.connect = connect
Alexandre's avatar
Alexandre committed
532
        self.multistreams = multistreams
533
534
        if self.multistreams:
            self.multi = self.create_multi()
535
            self.all_handles = []
Alexandre's avatar
Alexandre committed
536
        else:
537
            self.curl_handle = create_handle(self)
538

539
540
541
542
    def create_multi(self):
        multi = pycurl.CurlMulti()
        multi.setopt(pycurl.M_MAX_HOST_CONNECTIONS, 1)
        return multi
543

544
545
546
    def end(self):
        if not self.multistreams:
            self.curl_handle.close()
547
        else:
548
            self.remove_handles()
549
            self.multi.close()
550

551
552
553
554
555
556
557
    def remove_handles(self):
        n, handle_success, handle_fail = self.multi.info_read()
        handles = handle_success + handle_fail
        for h in handles:
            h.close()
            self.multi.remove_handle(h)

558
559
560
561
562
563
564
565
566
567
568
    def perform_multi(self):
        while 1:
            ret, num_handles = self.multi.perform()
            if ret != pycurl.E_CALL_MULTI_PERFORM:
                break
        while num_handles:
            ret = self.multi.select(1.0)
            if ret == -1:
                continue
            while 1:
                ret, num_handles = self.multi.perform()
Alexandre's avatar
Alexandre committed
569
570
571
572
                if not sync:
                    n, handle_pass, handle_fail = self.multi.info_read()
                    for handle in handle_pass:
                        self.read_result_handle(handle)
573
574
                if ret != pycurl.E_CALL_MULTI_PERFORM:
                    break
575
576
577
578
        if not sync:
            n, handle_pass, handle_fail = self.multi.info_read()
            for handle in handle_pass:
                self.read_result_handle(handle)
579

580
581
582
583
584
585
586
    def send(self, handle):
        handle.buffer = io.BytesIO()
        handle.setopt(pycurl.WRITEDATA, handle.buffer)
        try:
            handle.perform()
        except pycurl.error as e:
            error(e.args[1])
Alexandre's avatar
Alexandre committed
587

588
589
590
    def receive(self, handle):
        request = handle.request
        body = handle.buffer.getvalue()
591
        body_size = len(body)
592
        http_code = handle.getinfo(pycurl.RESPONSE_CODE)
Alexandre's avatar
Alexandre committed
593
594
        handle.time = handle.getinfo(pycurl.TOTAL_TIME)
        handle.pretime = handle.getinfo(pycurl.PRETRANSFER_TIME)
595
        try:
Alexandre's avatar
Alexandre committed
596
597
            content_type = handle.getinfo(pycurl.CONTENT_TYPE)
        except TypeError: # This is the exception we get if there is no Content-Type: (for intance in response to HEAD requests)
598
            content_type = None
599
600
        request.response = body
        request.response_size = body_size
601
602
        request.rcode = http_code
        request.ctype = content_type
603
        handle.buffer.close()
604

Alexandre's avatar
Alexandre committed
605
    def send_and_receive(self, handle, dump=False):
606
607
        self.send(handle)
        self.receive(handle)
608

609
610
611
    def read_result_handle(self, handle):
        self.receive(handle)
        handle.request.check_response()
612
613
        if show_time:
            print(f'{handle.request.i:3d}', end='   ')
614
            print(f'({handle.request.rcode})', end='   ')
615
616
617
            print(f'{handle.pretime * 1000:8.3f} ms', end='  ')
            print(f'{handle.time * 1000:8.3f} ms', end='  ')
            print(f'{(handle.time - handle.pretime) * 1000:8.3f} ms')
618
        if display_results:
619
620
621
            print("Return code %s (%.2f ms):" % (handle.request.rcode,
                (handle.time - handle.pretime) * 1000))
            print(f"{handle.request.response}\n")
Alexandre's avatar
Alexandre committed
622
623
        handle.close()
        self.multi.remove_handle(handle)
624
625
626
627
628

    def read_results(self):
        for handle in self.all_handles:
            self.read_result_handle(handle)

629
    def do_test(self, request, synchronous=True):
630
        if synchronous:
631
            handle = self.curl_handle
632
        else:
633
            handle = create_handle(self)
634
            self.all_handles.append(handle)
635
        handle.prepare(handle, self, request)
Alexandre's avatar
Alexandre committed
636
        if synchronous:
637
            self.send_and_receive(handle)
Alexandre's avatar
Alexandre committed
638
            request.check_response(self.debug)
639
        else:
640
            self.multi.add_handle(handle)
641
642
643
644
645
646
647
648
649
650
651
652
653


def get_next_domain(input_file):
    name, rtype = 'framagit.org', 'AAAA'
    line = input_file.readline()
    if line[:-1] == "":
        error("Not enough data in %s for the %i tests" % (ifile, tests))
    if line.find(' ') == -1:
        name = line[:-1]
        rtype = 'AAAA'
    else:
        (name, rtype) = line.split()
    return name, rtype
Alexandre's avatar
Alexandre committed
654

655
def print_result(connection, request, prefix=None, display_err=True):
Alexandre's avatar
Alexandre committed
656
    ok = request.ok
657
658
659
660
661
662
663
    dot = connection.dot
    server = connection.server
    rcode = request.rcode
    msg = request.response
    size = request.response_size
    if (dot and rcode) or (not dot and rcode == 200):
        if not monitoring:
664
            if display_results and (not check or verbose):
665
                print(msg)
666
        else:
667
            if expect is not None and expect not in str(request.response):
668
                ok = False
669
670
                print("%s Cannot find \"%s\" in response" % (server, expect))
                sys.exit(STATE_CRITICAL)
671
            if ok and size is not None and size > 0:
672
                print("%s OK - %s" % (server, "No error for %s/%s, %i bytes received" % (name, rtype, size)))
673
            else:
674
675
676
677
                print("%s OK - %s" % (server, "No error"))
            sys.exit(STATE_OK)
    else:
        if not monitoring:
678
            if display_err:
679
680
                if check:
                    print(connection.connect_to, end=': ', file=sys.stderr)
681
682
683
684
685
686
687
688
689
690
                if prefix:
                    print(prefix, end=': ', file=sys.stderr)
                if dot:
                    print("Error: %s" % msg, file=sys.stderr)
                else:
                   try:
                       msg = msg.decode()
                   except (UnicodeDecodeError, AttributeError):
                       pass # Sometimes, msg can be binary, or Latin-1
                   print("HTTP error %i: %s" % (rcode, msg), file=sys.stderr) 
691
        else:
692
693
            if not dot:
                print("%s HTTP error - %i: %s" % (server, rcode, msg))
694
            else:
695
696
697
698
699
                print("%s Error - %i: %s" % (server, rcode, msg))
            sys.exit(STATE_CRITICAL)
        ok = False
    return ok

700
def create_request(qname, qtype=rtype, use_edns=edns, want_dnssec=dnssec, dot=dot, trunc=False):
Alexandre's avatar
Alexandre committed
701
    if dot:
702
        request = RequestDoT(qname, rtype, use_edns, want_dnssec)
Alexandre's avatar
Alexandre committed
703
    else:
704
        request = RequestDoH(qname, rtype, use_edns, want_dnssec)
Alexandre's avatar
Alexandre committed
705
706
707
708
709
710
711
712
713
    if trunc:
        request.trunc_data()
    else:
        request.to_wire()
    return request

def create_requests_list(dot=dot, **req_args):
    requests = []
    if dot:
714
715
716
717
718
719
        requests.append(('Test 1', create_request(dot=dot, **req_args),
                        mandatory_levels["legal"]))
        requests.append(('Test 2', create_request(dot=dot, **req_args),
                         mandatory_levels["necessary"])) # RFC 7858,
        # section 3.3, SHOULD accept several requests on one connection.
        # TODO we miss the tests of pipelining and out-of-order.
Alexandre's avatar
Alexandre committed
720
    else:
721
722
723
724
725
726
727
728
        requests.append(('Test GET', create_request(**req_args), DOH_GET,
                         mandatory_levels["legal"])) # RFC 8484, section 4.1
        requests.append(('Test POST', create_request(**req_args), DOH_POST,
                         mandatory_levels["legal"])) # RFC 8484, section 4.1
        requests.append(('Test HEAD', create_request(**req_args), DOH_HEAD,
                         mandatory_levels["nicetohave"])) # HEAD
        # method is not mentioned in RFC 8484 (see section 4.1), so
        # just "nice to have".
Alexandre's avatar
Alexandre committed
729
    return requests
730

731
def run_check_default(connection):
Alexandre's avatar
Alexandre committed
732
    ok = True
733
734
735
736
    req_args = { 'qname': name, 'qtype': rtype, 'use_edns': edns, 'want_dnssec': dnssec }
    requests = create_requests_list(dot=dot, **req_args)
    for request_pack in requests:
        if dot:
737
            test_name, request, mandatory = request_pack
738
        else:
739
            test_name, request, method, mandatory = request_pack
740
741
        if verbose:
            print(test_name)
Alexandre's avatar
Alexandre committed
742
743
744
        if dot:
            bundle = request
        else:
745
746
747
748
            if method == DOH_POST:
                request.post = True
            elif method == DOH_HEAD:
                request.head = True
749
750
751
            handle = connection.curl_handle
            handle.prepare(handle, connection, request)
            bundle = handle
752
        try:
753
            connection.send_and_receive(bundle)
754
755
756
757
        except CustomException as e:
            ok = False
            error(e)
            break
Alexandre's avatar
Alexandre committed
758
        request.check_response(debug)
759
760
761
762
        if not print_result(connection, request, prefix=test_name, display_err=False):
            if mandatory >= mandatory_level:
                print_result(connection, request, prefix=test_name, display_err=True)
                ok = False
Alexandre's avatar
Alexandre committed
763
764
            if verbose:
                print()
765
            break
Alexandre's avatar
Alexandre committed
766
767
        if verbose:
            print()
Alexandre's avatar
Alexandre committed
768
    return ok
769

770
771
772
773
774
def run_check_mime(connection, accept="application/dns-message", content_type="application/dns-message"):
    if dot:
        return True
    ok = True
    header = [f"Accept: {accept}", f"Content-type: {content_type}"]
Alexandre's avatar
Alexandre committed
775
776
777
    if verbose:
        test_name = f'Test mime: {", ".join(h for h in header)}'
        print(test_name)
778
779
    req_args = { 'qname': name, 'qtype': rtype, 'use_edns': edns, 'want_dnssec': dnssec }
    request = create_request(**req_args)
780
781
782
    handle = connection.curl_handle
    handle.setopt(pycurl.HTTPHEADER, header)
    handle.prepare(handle, connection, request)
783
    try:
784
        connection.send_and_receive(handle)
785
786
787
    except CustomException as e:
        ok = False
        error(e)
Alexandre's avatar
Alexandre committed
788
    request.check_response(debug)
789
790
791
792
    if not print_result(connection, request, prefix=f"Test Header {', '.join(header)}"):
        ok = False
    default = "application/dns-message"
    default_header = [f"Accept: {default}", f"Content-type: {default}"]
793
    handle.setopt(pycurl.HTTPHEADER, default_header)
Alexandre's avatar
Alexandre committed
794
795
    if verbose:
        print()
796
797
    return ok

798
799
800
801
802
803
804
805
def run_check_trunc(connection):
    ok = True
    test_name = 'Test truncated data'
    if verbose:
        print(test_name)
    req_args = { 'qname': name, 'qtype': rtype, 'use_edns': edns, 'want_dnssec': dnssec }
    if dot:
        request = create_request(dot=dot, trunc=True, **req_args)
806
        bundle = request
807
808
809
    else:
        request = create_request(trunc=True, **req_args)
        request.post = True
810
811
812
        handle = connection.curl_handle
        handle.prepare(handle, connection, request)
        bundle = handle
813
    try:
814
        # 8.8.8.8 replies FORMERR but most DoT servers violently shut down the connection (which is legal)
Alexandre's avatar
Alexandre committed
815
        connection.send_and_receive(bundle, dump=debug)
816
817
818
    except CustomException as e:
        ok = False
        error(e)
819
820
    except OpenSSL.SSL.ZeroReturnError: # This is acceptable
        return ok
821
822
823
824
825
    except dns.exception.FormError: # This is also acceptable
        # Some DSN resolvers will echo mangled requests with
        # the RCODE set to FORMERR
        # so response can not be parsed in this case
        return ok
Alexandre's avatar
Alexandre committed
826
    if request.check_response(debug): # FORMERR is expected
Alexandre's avatar
Alexandre committed
827
828
829
830
        if dot:
            ok = request.rcode == dns.rcode.FORMERR
        else:
            ok = (request.response.rcode() == dns.rcode.FORMERR)
831
    else:
Alexandre's avatar
Alexandre committed
832
833
834
835
836
        if dot:
            ok = False
        else: # a 400 response's status is acceptable
            ok = (request.rcode >= 400 and request.rcode < 500)
    print_result(connection, request, prefix=test_name, display_err=not ok)
Alexandre's avatar
Alexandre committed
837
838
    if verbose:
        print()
839
840
    return ok

841
def run_check_additionals(connection):
842
843
    if not run_check_trunc(connection):
        return False
844
845
846
847
848
    # The DoH server is right to reject these (Example: 'HTTP
    # error 415: only Content-Type: application/dns-message is
    # supported')
    run_check_mime(connection, accept="text/html")
    run_check_mime(connection, content_type="text/html")
849
850
851
852
853
    return True

def run_check(connection):
    if not run_check_default(connection):
        return False
854
    if check_additional and not run_check_additionals(connection):
855
856
857
        return False
    return True

858
859
860
861
862
863
864
865
def resolved_ips(host, port, family, dot=dot):
    try:
        addr_list = socket.getaddrinfo(host, port, family)
    except socket.gaierror:
        error(f'Could not resolve "{url}"')
    ip_set = { addr[4][0] for addr in addr_list }
    return ip_set

866
# Main program
867
868
869
870
871
872
me = os.path.basename(sys.argv[0])
monitoring = (me == "check_doh" or me == "check_dot")
if not monitoring:
    name = None
    message = None
    try:
873
        optlist, args = getopt.getopt (sys.argv[1:], "hvPkeV:r:f:d:t46",
874
                                       ["help", "verbose", "debug", "dot", "head",
Alexandre's avatar
Alexandre committed
875
                                        "insecure", "POST", "vhost=", "multistreams",
876
                                        "sync", "no-display-results", "time",
877
                                        "dnssec", "noedns", "ecs", "repeat=", "file=", "delay=",
878
                                        "key=", "nosni",
879
                                        "v4only", "v6only",
880
                                        "check", "mandatory-level="])
881
882
883
884
        for option, value in optlist:
            if option == "--help" or option == "-h":
                usage()
                sys.exit(0)
885
886
            elif option == "--dot" or option == "-t":
                dot = True
887
888
            elif option == "--verbose" or option == "-v":
                verbose = True
Alexandre's avatar
Alexandre committed
889
            elif option == "--debug":
890
891
                debug = True
                verbose = True
892
            elif option == "--HEAD" or option == "--head" or option == "-e":
893
                head = True
894
            elif option == "--POST" or option == "--post" or option == "-P":
895
                post = True
896
897
            elif option == "--vhost" or option == "-V":
                vhostname = value
898
899
            elif option == "--insecure" or option == "-k":
                insecure = True
900
901
            elif option == "--multistreams":
                multistreams = True
Alexandre's avatar
Alexandre committed
902
903
            elif option == "--sync":
                sync = True
904
905
            elif option == "--no-display-results":
                display_results = False
906
907
            elif option == "--time":
                show_time = True
908
909
            elif option == "--dnssec":
                dnssec = True
910
911
            elif option == "--nosni":
                sni = False
<