homer.py 37.7 KB
Newer Older
Stephane Bortzmeyer's avatar
Stephane Bortzmeyer committed
1
2
#!/usr/bin/env python3

3
4
5
6
7
# Homer is a DoH (DNS-over-HTTPS) and DoT (DNS-over-TLS) client. Its
# main purpose is to test DoH and DoT resolvers. Reference site is
# <https://framagit.org/bortzmeyer/homer/> See author, documentation,
# etc, there, or in the README.md included with the distribution.

Stephane Bortzmeyer's avatar
Stephane Bortzmeyer committed
8
9
10
11
12
13
# http://pycurl.io/docs/latest
import pycurl

# http://www.dnspython.org/
import dns.message

14
15
16
# https://github.com/drkjam/netaddr/
import netaddr

17
18
19
20
21
# Octobre 2019: the Python GnuTLS bindings don't work with Python 3. So we use OpenSSL.
# https://www.pyopenssl.org/
# https://pyopenssl.readthedocs.io/
import OpenSSL

Stephane Bortzmeyer's avatar
Stephane Bortzmeyer committed
22
23
24
25
26
27
import io
import sys
import base64
import getopt
import urllib.parse
import time
28
29
import socket
import ctypes
30
import re
31
import os.path
32
33
import hashlib
import base64
34
import signal
Stephane Bortzmeyer's avatar
Stephane Bortzmeyer committed
35

36
# Values that can be changed from the command line
37
dot = False # DoH by default
Stephane Bortzmeyer's avatar
Stephane Bortzmeyer committed
38
verbose = False
39
debug = False
Stephane Bortzmeyer's avatar
Stephane Bortzmeyer committed
40
insecure = False
41
post = False
Stephane Bortzmeyer's avatar
Stephane Bortzmeyer committed
42
head = False
43
44
dnssec = False
edns = True
45
no_ecs = True
46
sni = True
Stephane Bortzmeyer's avatar
Stephane Bortzmeyer committed
47
rtype = 'AAAA'
48
vhostname = None
Stephane Bortzmeyer's avatar
Stephane Bortzmeyer committed
49
tests = 1 # Number of repeated tests
50
key = None # SPKI 
51
ifile = None # Input file
52
delay = None
53
54
55
forceIPv4 = False
forceIPv6 = False
connectTo = None
56
check = False
57
58
mandatory_level = None
check_additional = True
59
60
61
# Monitoring plugin only:
host = None
path = None
62
expect = None
Stephane Bortzmeyer's avatar
Stephane Bortzmeyer committed
63

64
65
66
# Do not change these
re_host = re.compile(r'^([0-9a-z][0-9a-z-\.]*)|([0-9:]+)|([0-9\.])$')

67
68
69
70
71
72
73
# For the monitoring plugin
STATE_OK = 0
STATE_WARNING = 1
STATE_CRITICAL = 2
STATE_UNKNOWN = 3
STATE_DEPENDENT = 4

Alexandre's avatar
Alexandre committed
74
75
76
77
# For the check option
DOH_GET = 0
DOH_POST = 1
DOH_HEAD = 2
78
79
# Is the test mandatory?
mandatory_levels = {"legal": 30, "necessary": 20, "nicetohave": 10}
Alexandre's avatar
Alexandre committed
80

81
82
TIMEOUT_CONN = 2

83
def error(msg=None, exit=True):
Stephane Bortzmeyer's avatar
Stephane Bortzmeyer committed
84
85
    if msg is None:
        msg = "Unknown error"
86
87
    if monitoring:
        print("%s: %s" % (url, msg))
88
89
        if exit:
            sys.exit(STATE_CRITICAL)
90
    else:
Alexandre's avatar
Alexandre committed
91
92
93
        print(msg, file=sys.stderr)
        if check:
            print('KO')
94
95
        if exit:
            sys.exit(1)
Alexandre's avatar
Alexandre committed
96

Stephane Bortzmeyer's avatar
Stephane Bortzmeyer committed
97
98
99
def usage(msg=None):
    if msg:
        print(msg,file=sys.stderr)
100
101
    print("Usage: %s [--dot] url-or-servername domain-name [DNS type]" % sys.argv[0], file=sys.stderr)
    print("See the README.md for more details.", file=sys.stderr)
Stephane Bortzmeyer's avatar
Stephane Bortzmeyer committed
102

103
def is_valid_hostname(name):
104
    name = canonicalize(name)
105
    return re_host.search(name)
106

107
108
109
110
111
112
113
114
115
def canonicalize(hostname):
    result = hostname.lower()
    # TODO handle properly the case where it fails with UnicodeError
    # (two consecutive dots for instance) to get a custom exception
    result = result.encode('idna').decode()
    if result[len(result)-1] == '.':
        result = result[:-1]
    return result

116
117
118
119
def is_valid_ip_address(addr):
    try:
        baddr = netaddr.IPAddress(addr)
    except netaddr.core.AddrFormatError:
120
121
        return (False, None)
    return (True, baddr.version)
122

123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
def is_valid_url(url):
  try:
    result = urllib.parse.urlparse(url) # A very poor validation, many
    # errors (for instance whitespaces, IPv6 address litterals without
    # brackets...) are ignored.
    return (result.scheme=="https" and result.netloc != "")
  except ValueError:
    return False

def get_certificate_san(x509cert):
    san = ""
    ext_count = x509cert.get_extension_count()
    for i in range(0, ext_count):
        ext = x509cert.get_extension(i)
        if "subjectAltName" in str(ext.get_short_name()):
            san = str(ext)
    return san

141
142
143
144
145
146
147
148
149
150
151
# Try one possible name. Names must be already canonicalized.
def match_hostname(hostname, possibleMatch):
    if possibleMatch.startswith("*."): # Wildcard
        base = possibleMatch[1:] # Skip the star
        # RFC 6125 says that we MAY accept left-most labels with
        # wildcards included (foo*bar). We don't do it here.
        try:
            (first, rest) = hostname.split(".", maxsplit=1)
        except ValueError: # One-label name
            rest = hostname
        if rest == base[1:]:
152
            return True
153
        if hostname == base[1:]:
154
            return True
155
156
157
158
159
160
161
162
        return False
    else:
        return hostname == possibleMatch

# Try all the names in the certificate
def validate_hostname(hostname, cert):
    # Complete specification is in RFC 6125. It is long and
    # complicated and I'm not sure we do it perfectly.
163
    (is_addr, family) = is_valid_ip_address(hostname)
164
    hostname = canonicalize(hostname)
165
    for alt_name in get_certificate_san(cert).split(", "):
166
        if alt_name.startswith("DNS:") and not is_addr:
167
            (start, base) = alt_name.split("DNS:")
168
            base = canonicalize(base)
169
170
            found = match_hostname(hostname, base)
            if found:
171
                return True
172
173
        elif alt_name.startswith("IP Address:") and is_addr:
            host_i = netaddr.IPAddress(hostname)
174
175
176
            (start, base) = alt_name.split("IP Address:")
            if base.endswith("\n"):
                base = base[:-1]
177
178
179
180
            try:
                base_i = netaddr.IPAddress(base)
            except netaddr.core.AddrFormatError:
                continue # Ignore broken IP addresses in certificates. Are we too liberal?
Alexandre's avatar
Alexandre committed
181
            if host_i == base_i:
182
183
                return True
        else:
184
185
186
            pass # Ignore unknown alternative name types. May be
                 # accept URI alternative names for DoH,
    # According to RFC 6125, we MUST NOT try the Common Name before the Subject Alternative Names.
187
    cn = canonicalize(cert.get_subject().commonName)
188
189
190
    found = match_hostname(hostname, cn)
    if found:
        return True
191
192
    return False

193
194
195
196
197
198
def dump_data(data, text="data"):
    pref = ' ' * (len(text) - 4)
    print(f'{text}: ', data)
    print(pref, 'hex:', " ".join(format(c, '02x') for c in data))
    print(pref, 'bin:', " ".join(format(c, '08b') for c in data))

199
200
201
202
203
204
def timeout_connection(signum, frame):
    raise TimeoutConnectionError('Connection timeout')

class TimeoutConnectionError(Exception):
    pass

205

Alexandre's avatar
Alexandre committed
206
207
208
209
class CustomException(Exception):
    pass


Alexandre's avatar
Alexandre committed
210
class Request:
211
    def __init__(self, qname, qtype=rtype, use_edns=edns, want_dnssec=dnssec):
212
213
214
215
216
        if no_ecs:
             opt = dns.edns.ECSOption(address='', srclen=0) # Disable ECS (RFC 7871, section 7.1.2)
             options = [opt]
        else:
            options = None
217
        self.message = dns.message.make_query(qname, dns.rdatatype.from_text(qtype),
218
                                              use_edns=use_edns, want_dnssec=want_dnssec, options=options)
219
        self.message.flags |= dns.flags.AD # Ask for validation
Alexandre's avatar
Alexandre committed
220
        self.ok = True
Alexandre's avatar
Alexandre committed
221

222
223
224
225
226
    def trunc_data(self):
        self.data = self.message.to_wire()
        half = round(len(self.data) / 2)
        self.data = self.data[:half]

Alexandre's avatar
Alexandre committed
227
    def to_wire(self):
228
229
        self.data = self.message.to_wire()

Alexandre's avatar
Alexandre committed
230
231

class RequestDoT(Request):
Alexandre's avatar
Alexandre committed
232
    def check_response(self, debug=False):
233
        ok = self.ok
234
        if not self.rcode:
235
236
            self.ok = False
            return False
237
        if self.response.id != self.message.id:
238
            self.response = "The ID in the answer does not match the one in the query"
Alexandre's avatar
Alexandre committed
239
240
            if debug:
                self.response += f'"(query id: {self.message.id}) (response id: {self.response.id})'
241
242
243
            self.ok = False
            return False
        return self.ok
244
245


Alexandre's avatar
Alexandre committed
246
class RequestDoH(Request):
247
    def __init__(self, qname, qtype=rtype, use_edns=edns, want_dnssec=dnssec):
Alexandre's avatar
Alexandre committed
248
        Request.__init__(self, qname, qtype=rtype, use_edns=edns, want_dnssec=dnssec)
249
        self.message.id = 0 # DoH requests that
250
        self.post = False
Alexandre's avatar
Alexandre committed
251
        self.head = False
252

Alexandre's avatar
Alexandre committed
253
    def check_response(self, debug=False):
Alexandre's avatar
Alexandre committed
254
        ok = self.ok
255
        if self.rcode == 200:
256
257
            if self.ctype != "application/dns-message":
                self.response = "Content type of the response (\"%s\") invalid" % self.ctype
258
259
                ok = False
            else:
Alexandre's avatar
Alexandre committed
260
                if not self.head:
261
                    try:
Alexandre's avatar
Alexandre committed
262
                        response = dns.message.from_wire(self.response)
263
264
265
                    except dns.message.TrailingJunk: # Not DNS. Should
                        # not happen for a content type
                        # application/dns-message but who knows?
Alexandre's avatar
Alexandre committed
266
267
268
                        self.response = "ERROR Not proper DNS data, trailing junk"
                        if debug:
                            self.response += " \"%s\"" % response
269
270
                        ok = False
                    except dns.name.BadLabelType: # Not DNS.
Alexandre's avatar
Alexandre committed
271
272
273
                        self.response = "ERROR Not proper DNS data (wrong path in the URL?)"
                        if debug:
                            self.response += " \"%s\"" % response[:100]
274
                        ok = False
Alexandre's avatar
Alexandre committed
275
276
                    else:
                        self.response = response
277
                else:
Alexandre's avatar
Alexandre committed
278
                    if self.response_size == 0:
279
280
                        self.response = "HEAD successful"
                    else:
Alexandre's avatar
Alexandre committed
281
282
283
284
                        data = self.response
                        self.response = "ERROR Body length is not null"
                        if debug:
                            self.response += "\"%s\"" % data[:100]
285
286
287
                        ok = False
        else:
            ok = False
Alexandre's avatar
Alexandre committed
288
            if self.response_size == 0:
289
290
                self.response = "[No details]"
            else:
291
                self.response = self.response
Alexandre's avatar
Alexandre committed
292
        self.ok = ok
Alexandre's avatar
Alexandre committed
293
        return ok
294

295

296
class Connection:
297
    def __init__(self, server, servername=None, connect=None, forceIPv4=False, forceIPv6=False,
298
                 dot=dot, verbose=verbose, debug=debug, insecure=insecure):
299
        if dot and not is_valid_hostname(server):
300
            error("DoT requires a host name or IP address, not \"%s\"" % server)
301
        if not dot and not is_valid_url(server):
302
            error("DoH requires a valid HTTPS URL, not \"%s\"" % server)
303
        if forceIPv4 and forceIPv6:
Alexandre's avatar
Alexandre committed
304
            raise CustomException("Force IPv4 *or* IPv6 but not both")
305
        self.dot = dot
306
        self.server = server
307
308
        self.servername = servername
        if self.servername is not None:
Alexandre's avatar
Alexandre committed
309
            self.check = self.servername
310
        else:
Alexandre's avatar
Alexandre committed
311
            self.check = self.server
312
313
        self.dot = dot
        self.verbose = verbose
314
        self.debug = debug
315
        self.insecure = insecure
316
        self.connect_to = connect
317

318
319
    def __str__(self):
        return self.server
320

321
322
323
    def check_ip_address(self, addr):
        (is_addr, self.family) = is_valid_ip_address(addr)
        if not is_addr and not self.dot:
Alexandre's avatar
Alexandre committed
324
            raise CustomException("%s is not IPv4 and not IPv6" % addr)
325
        if forceIPv4 and self.family == 6:
Alexandre's avatar
Alexandre committed
326
            raise CustomException("You cannot force IPv4 with a litteral IPv6 address (%s)" % addr)
327
        elif forceIPv6 and self.family == 4:
Alexandre's avatar
Alexandre committed
328
            raise CustomException("You cannot force IPv6 with a litteral IPv4 address (%s)" % addr)
329
330
331
332
333
334
335
336
337
        if forceIPv4 or self.family == 4:
            self.family = socket.AF_INET
            self.repraddress = addr
        elif forceIPv6 or self.family == 6:
            self.family = socket.AF_INET6
            self.repraddress = f'[{addr}]'
        else:
            self.family = 0

Alexandre's avatar
Alexandre committed
338
    def do_test(self, qname, qtype=rtype):
339
        # Routine doing one actual test. Returns a Request object
Alexandre's avatar
Alexandre committed
340
341
        pass

342
343
344

class ConnectionDoT(Connection):
    def __init__(self, server, servername=None, connect=None, forceIPv4=False, forceIPv6=False,
345
                 verbose=verbose, debug=debug, insecure=insecure):
346
        Connection.__init__(self, server, servername=servername, connect=connect,
347
                forceIPv4=forceIPv4, forceIPv6=forceIPv6, dot=True,
348
                verbose=verbose, debug=debug, insecure=insecure)
349
350
351
352
353
        if connect is not None:
            addr = connect
        else:
            addr = self.server
        self.check_ip_address(addr)
Alexandre's avatar
Alexandre committed
354
        addrinfo_list = socket.getaddrinfo(addr, 853, self.family)
355
356
        addrinfo_set = { (addrinfo[4], addrinfo[0]) for addrinfo in addrinfo_list }
        signal.signal(signal.SIGALRM, timeout_connection)
Alexandre's avatar
Alexandre committed
357
        self.success = False
358
359
        for addrinfo in addrinfo_set:
            self.hasher = hashlib.sha256()
360
            if self.connect(addrinfo[0], addrinfo[1]):
Alexandre's avatar
Alexandre committed
361
                self.success = True
362
                break
Alexandre's avatar
Alexandre committed
363
            if self.verbose and connect is None:
Alexandre's avatar
Alexandre committed
364
                print("Trying another IP address")
Alexandre's avatar
Alexandre committed
365
366
        if not self.success:
            if self.verbose and connect is None:
Alexandre's avatar
Alexandre committed
367
                print("No other IP address")
Alexandre's avatar
Alexandre committed
368
369
370
            if connect is None:
                error(f'Could not connect to "{server}"')
            else:
371
                print(f'Could not connect to "{server}" on {connect}')
Alexandre's avatar
Alexandre committed
372

373
374
375
376
377

    def connect(self, addr, sock_family):
        signal.alarm(TIMEOUT_CONN)
        self.addr = addr
        self.sock = socket.socket(sock_family, socket.SOCK_STREAM)
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
        if self.verbose:
            print("Connecting to %s ..." % str(self.addr))
        # With typical DoT servers, we *must* use TLS 1.2 (otherwise,
        # do_handshake fails with "OpenSSL.SSL.SysCallError: (-1, 'Unexpected
        # EOF')" Typical HTTP servers are more lax.
        self.context = OpenSSL.SSL.Context(OpenSSL.SSL.TLSv1_2_METHOD)
        if self.insecure:
            self.context.set_verify(OpenSSL.SSL.VERIFY_NONE, lambda *x: True)
        else:
            self.context.set_default_verify_paths()
            self.context.set_verify_depth(4) # Seems ignored
            self.context.set_verify(OpenSSL.SSL.VERIFY_PEER | OpenSSL.SSL.VERIFY_FAIL_IF_NO_PEER_CERT | \
                                    OpenSSL.SSL.VERIFY_CLIENT_ONCE,
                                    lambda conn, cert, errno, depth, preverify_ok: preverify_ok)
        self.session = OpenSSL.SSL.Connection(self.context, self.sock)
393
394
        if sni:
            self.session.set_tlsext_host_name(canonicalize(self.check).encode()) # Server Name Indication (SNI)
395
396
397
398
399
        try:
            self.session.connect((self.addr))
            # TODO We may here have exceptions such as OpenSSL.SSL.ZeroReturnError
            self.session.do_handshake()
        except TimeoutConnectionError:
Alexandre's avatar
Alexandre committed
400
401
            if self.verbose:
                print("Timeout")
402
            return False
403
404
405
406
        except OSError:
            if self.verbose:
                print("Cannot connect")
            return False
Alexandre's avatar
Alexandre committed
407
408
409
410
        except OpenSSL.SSL.Error as e:
            if self.verbose:
                print(f"OpenSSL error: {', '.join(err[0][2] for err in e.args)}")
            return False
411
        # RFC 7858, section 4.2 and appendix A
412
        self.cert = self.session.get_peer_certificate()
413
        self.publickey = self.cert.get_pubkey()
414
        if debug or key is not None:
415
416
417
418
            self.hasher.update(OpenSSL.crypto.dump_publickey(OpenSSL.crypto.FILETYPE_ASN1,
                                                  self.publickey))
            self.digest = self.hasher.digest()
            key_string = base64.standard_b64encode(self.digest).decode()
419
        if debug:
420
421
422
423
424
            print("Certificate #%x for \"%s\", delivered by \"%s\"" % \
                  (self.cert.get_serial_number(),
                   self.cert.get_subject().commonName,
                   self.cert.get_issuer().commonName))
            print("Public key is pin-sha256=\"%s\"" % \
425
                  key_string)
426
        if not insecure:
427
428
429
430
431
432
433
            if key is None:
                valid = validate_hostname(self.check, self.cert)
                if not valid:
                    error("Certificate error: \"%s\" is not in the certificate" % (self.check))
            else:
                if key_string != key:
                    error("Key error: expected \"%s\", got \"%s\"" % (key, key_string))
434
        signal.alarm(0)
435
        return True
436

437
    def end(self):
438
439
440
        self.session.shutdown()
        self.session.close()

441
442
443
    def send_data(self, data, dump=False):
        if dump:
            dump_data(data, 'data sent')
444
445
446
        length = len(data)
        self.session.send(length.to_bytes(2, byteorder='big') + data)

447
    def receive_data(self, request, dump=False):
Alexandre's avatar
Alexandre committed
448
        buf = self.session.recv(2)
449
450
        request.response_size = int.from_bytes(buf, byteorder='big')
        buf = self.session.recv(request.response_size)
451
452
        if dump:
            dump_data(buf, 'data recv')
453
454
        request.response = dns.message.from_wire(buf)
        request.rcode = True
455

456
457
458
    def send_and_receive(self, request, dump=False):
        self.send_data(request.data, dump=dump)
        self.receive_data(request, dump=dump)
459
460

    def do_test(self, qname, qtype=rtype):
461
        request = RequestDoT(qname, qtype, want_dnssec=dnssec, use_edns=edns)
462
        request.to_wire()
463
        self.send_and_receive(request)
Alexandre's avatar
Alexandre committed
464
        request.check_response(self.debug)
465
        return request
Alexandre's avatar
Alexandre committed
466

467
468
469

class ConnectionDoH(Connection):
    def __init__(self, server, servername=None, connect=None, forceIPv4=False, forceIPv6=False,
470
                 verbose=verbose, debug=debug, insecure=insecure):
471
        Connection.__init__(self, server, servername=servername, connect=connect,
472
                forceIPv4=forceIPv4, forceIPv6=forceIPv6, dot=False,
473
                verbose=verbose, debug=debug, insecure=insecure)
474
        self.url = server
Alexandre's avatar
Alexandre committed
475
476
        self.connect = connect

477
478
479
480
481
    def create_handle(self):
        self.curl = pycurl.Curl()
        # Does not work if pycurl was not compiled with nghttp2 (recent Debian
        # packages are OK) https://github.com/pycurl/pycurl/issues/477
        self.curl.setopt(pycurl.HTTP_VERSION, pycurl.CURL_HTTP_VERSION_2)
482
        if self.debug:
483
            self.curl.setopt(pycurl.VERBOSE, True)
484
        if self.insecure:
485
486
            self.curl.setopt(pycurl.SSL_VERIFYPEER, False)
            self.curl.setopt(pycurl.SSL_VERIFYHOST, False)
487
        if forceIPv4:
488
            self.curl.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_V4)
489
        if forceIPv6:
490
            self.curl.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_V6)
Alexandre's avatar
Alexandre committed
491
492
        if self.connect is not None:
            self.check_ip_address(self.connect)
493
            self.curl.setopt(pycurl.CONNECT_TO, [f'::{self.repraddress}:443',])
Alexandre's avatar
Alexandre committed
494
        self.curl.setopt(pycurl.HTTPHEADER, ["Accept: application/dns-message", "Content-type: application/dns-message"])
495
496

    def end(self):
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
        self.curl.close()

    def set_opt(self, opt, value):
        self.curl.setopt(opt, value)

    def reset_opt_default(self):
        opts = {
                pycurl.NOBODY: False,
                pycurl.POST: False,
                pycurl.POSTFIELDS: '',
                pycurl.URL: ''
               }
        for opt, value in opts.items():
            self.set_opt(opt, value)

    def prepare(self, request):
        try:
            self.reset_opt_default()
        except AttributeError:
            self.create_handle()
517
        if request.post:
518
            self.prepare_post(request)
519
        elif request.head:
520
521
522
            self.prepare_head(request)
        else:
            self.prepare_get(request)
523

524
525
526
527
    def prepare_get(self, request):
        self.set_opt(pycurl.HTTPGET, True)
        dns_req = base64.urlsafe_b64encode(request.data).decode('UTF8').rstrip('=')
        self.set_opt(pycurl.URL, self.server + ("?dns=%s" % dns_req))
528

529
    def prepare_post(self, request):
530
        request.post = True
531
532
533
        self.set_opt(pycurl.POST, True)
        self.set_opt(pycurl.POSTFIELDS, request.data)
        self.set_opt(pycurl.URL, self.server)
534

535
    def prepare_head(self, request):
536
        request.head = True
537
538
539
540
541
542
        self.prepare_get(request)
        self.set_opt(pycurl.NOBODY, True)

    def perform(self):
        self.buffer = io.BytesIO()
        self.set_opt(pycurl.WRITEDATA, self.buffer)
Alexandre's avatar
Alexandre committed
543
544
545
546
        try:
            self.curl.perform()
        except pycurl.error as e:
            error(e.args[1])
547
548
549
550
551

    def receive(self, request):
        body = self.buffer.getvalue()
        body_size = len(body)
        http_code = self.curl.getinfo(pycurl.RESPONSE_CODE)
552
553
554
555
        try:
            content_type = self.curl.getinfo(pycurl.CONTENT_TYPE)
        except TypeError: # This is the exception we get if there is no Content-Type: (for intance in rsponse to HEAD requests)
            content_type = None
556
        request.response = body
Alexandre's avatar
Alexandre committed
557
        request.response_size = body_size
558
559
560
561
        request.rcode = http_code
        request.ctype = content_type
        self.buffer.close()

562
    def send_and_receive(self, request, dump=False):
563
564
565
        self.prepare(request)
        self.perform()
        self.receive(request)
566

Alexandre's avatar
Alexandre committed
567
    def do_test(self, qname, qtype=rtype):
568
        request = RequestDoH(qname, qtype, want_dnssec=dnssec, use_edns=edns)
569
570
        request.head = head
        request.post = post
571
        request.to_wire()
572
        self.send_and_receive(request)
Alexandre's avatar
Alexandre committed
573
        request.check_response(self.debug)
574
        return request
Alexandre's avatar
Alexandre committed
575

Alexandre's avatar
Alexandre committed
576
577
578
579
580
581
582
583
584
585
586
587
588

def get_next_domain(input_file):
    name, rtype = 'framagit.org', 'AAAA'
    line = input_file.readline()
    if line[:-1] == "":
        error("Not enough data in %s for the %i tests" % (ifile, tests))
    if line.find(' ') == -1:
        name = line[:-1]
        rtype = 'AAAA'
    else:
        (name, rtype) = line.split()
    return name, rtype

589
def print_result(connection, request, prefix=None, display_err=True):
Alexandre's avatar
Alexandre committed
590
    ok = request.ok
591
592
593
594
595
596
597
598
599
600
    dot = connection.dot
    server = connection.server
    rcode = request.rcode
    msg = request.response
    size = request.response_size
    if (dot and rcode) or (not dot and rcode == 200):
        if not monitoring:
            if not check or verbose:
                print(msg)
        else:
601
602
603
604
            if expect is not None and expect not in str(request.response):
                ok = False
                print("%s Cannot find \"%s\" in response" % (server, expect))
                sys.exit(STATE_CRITICAL)
605
            if ok and size is not None and size > 0:
606
607
608
609
610
611
                print("%s OK - %s" % (server, "No error for %s/%s, %i bytes received" % (name, rtype, size)))
            else:
                print("%s OK - %s" % (server, "No error"))
            sys.exit(STATE_OK)
    else:
        if not monitoring:
612
            if display_err:
613
614
                if check:
                    print(connection.connect_to, end=': ', file=sys.stderr)
615
616
617
618
619
620
621
622
623
624
                if prefix:
                    print(prefix, end=': ', file=sys.stderr)
                if dot:
                    print("Error: %s" % msg, file=sys.stderr)
                else:
                   try:
                       msg = msg.decode()
                   except (UnicodeDecodeError, AttributeError):
                       pass # Sometimes, msg can be binary, or Latin-1
                   print("HTTP error %i: %s" % (rcode, msg), file=sys.stderr) 
625
626
627
628
629
630
631
632
633
        else:
            if not dot:
                print("%s HTTP error - %i: %s" % (server, rcode, msg))
            else:
                print("%s Error - %i: %s" % (server, rcode, msg))
            sys.exit(STATE_CRITICAL)
        ok = False
    return ok

Alexandre's avatar
Alexandre committed
634
635
636
637
638
639
640
641
642
643
644
645
646
647
def create_request(dot=dot, trunc=False, **req_args):
    if dot:
        request = RequestDoT(**req_args)
    else:
        request = RequestDoH(**req_args)
    if trunc:
        request.trunc_data()
    else:
        request.to_wire()
    return request

def create_requests_list(dot=dot, **req_args):
    requests = []
    if dot:
648
649
650
651
652
653
        requests.append(('Test 1', create_request(dot=dot, **req_args),
                        mandatory_levels["legal"]))
        requests.append(('Test 2', create_request(dot=dot, **req_args),
                         mandatory_levels["necessary"])) # RFC 7858,
        # section 3.3, SHOULD accept several requests on one connection.
        # TODO we miss the tests of pipelining and out-of-order.
Alexandre's avatar
Alexandre committed
654
    else:
655
656
657
658
659
660
661
662
        requests.append(('Test GET', create_request(**req_args), DOH_GET,
                         mandatory_levels["legal"])) # RFC 8484, section 4.1
        requests.append(('Test POST', create_request(**req_args), DOH_POST,
                         mandatory_levels["legal"])) # RFC 8484, section 4.1
        requests.append(('Test HEAD', create_request(**req_args), DOH_HEAD,
                         mandatory_levels["nicetohave"])) # HEAD
        # method is not mentioned in RFC 8484 (see section 4.1), so
        # just "nice to have".
Alexandre's avatar
Alexandre committed
663
    return requests
664

665
def run_check_default(connection):
Alexandre's avatar
Alexandre committed
666
    ok = True
667
668
669
670
    req_args = { 'qname': name, 'qtype': rtype, 'use_edns': edns, 'want_dnssec': dnssec }
    requests = create_requests_list(dot=dot, **req_args)
    for request_pack in requests:
        if dot:
671
            test_name, request, mandatory = request_pack
672
        else:
673
            test_name, request, method, mandatory = request_pack
674
675
676
677
678
679
680
681
682
683
684
685
686
        if verbose:
            print(test_name)
        if not dot:
            if method == DOH_POST:
                request.post = True
            elif method == DOH_HEAD:
                request.head = True
        try:
            connection.send_and_receive(request)
        except CustomException as e:
            ok = False
            error(e)
            break
Alexandre's avatar
Alexandre committed
687
        request.check_response(debug)
688
689
690
691
        if not print_result(connection, request, prefix=test_name, display_err=False):
            if mandatory >= mandatory_level:
                print_result(connection, request, prefix=test_name, display_err=True)
                ok = False
Alexandre's avatar
Alexandre committed
692
693
            if verbose:
                print()
694
            break
Alexandre's avatar
Alexandre committed
695
696
        if verbose:
            print()
Alexandre's avatar
Alexandre committed
697
    return ok
698

699
700
701
702
703
def run_check_mime(connection, accept="application/dns-message", content_type="application/dns-message"):
    if dot:
        return True
    ok = True
    header = [f"Accept: {accept}", f"Content-type: {content_type}"]
Alexandre's avatar
Alexandre committed
704
705
706
    if verbose:
        test_name = f'Test mime: {", ".join(h for h in header)}'
        print(test_name)
707
708
709
710
711
712
713
714
    req_args = { 'qname': name, 'qtype': rtype, 'use_edns': edns, 'want_dnssec': dnssec }
    request = create_request(**req_args)
    connection.curl.setopt(pycurl.HTTPHEADER, header)
    try:
        connection.send_and_receive(request)
    except CustomException as e:
        ok = False
        error(e)
Alexandre's avatar
Alexandre committed
715
    request.check_response(debug)
716
717
718
719
720
    if not print_result(connection, request, prefix=f"Test Header {', '.join(header)}"):
        ok = False
    default = "application/dns-message"
    default_header = [f"Accept: {default}", f"Content-type: {default}"]
    connection.curl.setopt(pycurl.HTTPHEADER, default_header)
Alexandre's avatar
Alexandre committed
721
722
    if verbose:
        print()
723
724
    return ok

725
726
727
728
729
730
731
732
733
734
735
736
def run_check_trunc(connection):
    ok = True
    test_name = 'Test truncated data'
    if verbose:
        print(test_name)
    req_args = { 'qname': name, 'qtype': rtype, 'use_edns': edns, 'want_dnssec': dnssec }
    if dot:
        request = create_request(dot=dot, trunc=True, **req_args)
    else:
        request = create_request(trunc=True, **req_args)
        request.post = True
    try:
737
        # 8.8.8.8 replies FORMERR but most DoT servers violently shut down the connection (which is legal)
738
        connection.send_and_receive(request, dump=debug)
739
740
741
    except CustomException as e:
        ok = False
        error(e)
742
743
    except OpenSSL.SSL.ZeroReturnError: # This is acceptable
        return ok
744
745
746
747
748
    except dns.exception.FormError: # This is also acceptable
        # Some DSN resolvers will echo mangled requests with
        # the RCODE set to FORMERR
        # so response can not be parsed in this case
        return ok
Alexandre's avatar
Alexandre committed
749
    if request.check_response(debug): # FORMERR is expected
Alexandre's avatar
Alexandre committed
750
751
752
753
        if dot:
            ok = request.rcode == dns.rcode.FORMERR
        else:
            ok = (request.response.rcode() == dns.rcode.FORMERR)
754
    else:
Alexandre's avatar
Alexandre committed
755
756
757
758
759
        if dot:
            ok = False
        else: # a 400 response's status is acceptable
            ok = (request.rcode >= 400 and request.rcode < 500)
    print_result(connection, request, prefix=test_name, display_err=not ok)
Alexandre's avatar
Alexandre committed
760
761
    if verbose:
        print()
762
763
    return ok

764
def run_check_additionals(connection):
765
766
    if not run_check_trunc(connection):
        return False
767
768
769
770
771
    # The DoH server is right to reject these (Example: 'HTTP
    # error 415: only Content-Type: application/dns-message is
    # supported')
    run_check_mime(connection, accept="text/html")
    run_check_mime(connection, content_type="text/html")
772
773
774
775
776
    return True

def run_check(connection):
    if not run_check_default(connection):
        return False
777
    if check_additional and not run_check_additionals(connection):
778
779
780
        return False
    return True

781
782
783
784
785
786
787
788
def resolved_ips(host, port, family, dot=dot):
    try:
        addr_list = socket.getaddrinfo(host, port, family)
    except socket.gaierror:
        error(f'Could not resolve "{url}"')
    ip_set = { addr[4][0] for addr in addr_list }
    return ip_set

789
# Main program
790
791
792
793
794
795
me = os.path.basename(sys.argv[0])
monitoring = (me == "check_doh" or me == "check_dot")
if not monitoring:
    name = None
    message = None
    try:
796
797
        optlist, args = getopt.getopt (sys.argv[1:], "hvdPkeV:r:f:d:t46",
                                       ["help", "verbose", "debug", "dot", "head",
798
                                        "insecure", "POST", "vhost=",
799
                                        "dnssec", "noedns", "ecs", "repeat=", "file=", "delay=",
800
                                        "key=", "nosni",
801
                                        "v4only", "v6only",
802
                                        "check", "mandatory-level="])
803
804
805
806
        for option, value in optlist:
            if option == "--help" or option == "-h":
                usage()
                sys.exit(0)
807
808
            elif option == "--dot" or option == "-t":
                dot = True
809
810
            elif option == "--verbose" or option == "-v":
                verbose = True
811
812
813
            elif option == "--debug" or option == "-d":
                debug = True
                verbose = True
814
            elif option == "--HEAD" or option == "--head" or option == "-e":
815
                head = True
816
            elif option == "--POST" or option == "--post" or option == "-P":
817
                post = True
818
819
            elif option == "--vhost" or option == "-V":
                vhostname = value
820
821
            elif option == "--insecure" or option == "-k":
                insecure = True
822
823
            elif option == "--dnssec":
                dnssec = True
824
825
            elif option == "--nosni":
                sni = False
826
827
828
829
            elif option == "--noedns": # Warning: it will mean the
                                       # resolver may send ECS
                                       # information to the
                                       # authoritative name servers.
830
                edns = False
831
832
            elif option == "--ecs":
                no_ecs = False
833
834
835
836
837
838
839
840
841
842
            elif option == "--repeat" or option == "-r":
                tests = int(value)
                if tests <= 1:
                    error("--repeat needs a value > 1")
            elif option == "--delay" or option == "-d":
                delay = float(value)
                if delay <= 0:
                    error("--delay needs a value > 0")
            elif option == "--file" or option == "-f":
                ifile = value
843
844
            elif option == "--key":
                key = value
845
846
847
848
            elif option == "-4" or option == "v4only":
                forceIPv4 = True
            elif option == "-6" or option == "v6only":
                forceIPv6 = True
849
850
            elif option == "--check":
                check = True
851
852
            elif option == "--mandatory-level":
                mandatory_level = value
853
854
855
856
857
858
859
860
861
862
863
864
            else:
                error("Unknown option %s" % option)
    except getopt.error as reason:
        usage(reason)
        sys.exit(1)
    if tests <= 1 and delay is not None:
        error("--delay makes no sense if there is no repetition")
    if post and head:
        usage("POST or HEAD but not both")
        sys.exit(1)
    if dot and (post or head):
        usage("POST or HEAD makes non sense for DoT")
Alexandre's avatar
Alexandre committed
865
        sys.exit(1)
866
867
868
    if not edns and not no_ecs:
        usage("ECS requires EDNS")
        sys.exit(1)
869
870
871
872
873
874
875
876
877
878
    if mandatory_level is not None and \
       mandatory_level not in mandatory_levels.keys():
        usage("Unknown mandatory level \"%s\"" % mandatory_level)
        sys.exit(1)
    if mandatory_level is not None and not check:
        usage("--mandatory-level only makes sense with --check")
        sys.exit(1)
    if mandatory_level is None:
        mandatory_level = "necessary"
    mandatory_level = mandatory_levels[mandatory_level]
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
    if ifile is None and (len(args) != 2 and len(args) != 3):
        usage("Wrong number of arguments")
        sys.exit(1)
    if ifile is not None and len(args) != 1:
        usage("Wrong number of arguments (if --file is used, do not indicate the domain name)")
        sys.exit(1)
    url = args[0]
    if ifile is None:
        name = args[1]
        if len(args) == 3:
            rtype = args[2]
else: # Monitoring plugin
    dot = (me == "check_dot")
    name = None
    try:
894
        optlist, args = getopt.getopt (sys.argv[1:], "H:n:p:V:t:e:Pih46k:x")
895
896
897
898
        for option, value in optlist:
            if option == "-H":
                host = value
            elif option == "-V":
Alexandre's avatar
Alexandre committed
899
                vhostname = value
900
901
902
903
            elif option == "-n":
                name = value
            elif option == "-t":
                rtype = value
904
905
            elif option == "-e":
                expect = value
906
907
908
909
910
911
            elif option == "-p":
                path = value
            elif option == "-P":
                post = True
            elif option == "-h":
                head = True
912
913
            elif option == "-i":
                insecure = True
914
915
            elif option == "-x":
                sni = False
916
917
918
919
            elif option == "-4":
                forceIPv4 = True
            elif option == "-6":
                forceIPv6 = True
920
921
            elif option == "-k":
                key = value
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
            else:
                # Should never occur, it is trapped by getopt
                print("Unknown option %s" % option)
                sys.exit(STATE_UNKNOWN)
    except getopt.error as reason:
        print("Option parsing problem %s" % reason)
        sys.exit(STATE_UNKNOWN)
    if len(args) > 0:
        print("Too many arguments (\"%s\")" % args)
        sys.exit(STATE_UNKNOWN)
    if host is None or name is None:
        print("Host (-H) and name to lookup (-n) are necessary")
        sys.exit(STATE_UNKNOWN)
    if post and head:
        print("POST or HEAD but not both")
        sys.exit(STATE_UNKNOWN)
938
939
940
941
942
943
    if dot and (post or head):
        print("POST or HEAD makes no sense for DoT")
        sys.exit(STATE_UNKNOWN)
    if dot and path:
        print("URL path makes no sense for DoT")
        sys.exit(STATE_UNKNOWN)
944
945
946
    if dot:
        url = host
    else:
947
948
        if vhostname is None or vhostname == host:
            connectTo = None
949
            url = "https://%s/" % host
Stephane Bortzmeyer's avatar
Stephane Bortzmeyer committed
950
        else:
951
            connectTo = host
Alexandre's avatar
Alexandre committed
952
            url = "https://%s/" % vhostname
953
954
955
956
        if path is not None:
            if path.startswith("/"):
                path = path[1:]
            url += path
957

958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
# retrieve all ips when using --check
# not necessary if connectTo is already defined
# as it is the case with --monitoring
if not check or connectTo is not None:
    ip_set = {connectTo, }
else:
    if dot:
        port = 853
        if not is_valid_hostname(url):
            error("DoT requires a host name or IP address, not \"%s\"" % url)
        netloc = url
    else:
        port = 443
        if not is_valid_url(url):
            error("DoH requires a valid HTTPS URL, not \"%s\"" % url)
Alexandre's avatar
Alexandre committed
973
974
975
976
977
978
        try:
            url_parts = urllib.parse.urlparse(url) # A very poor validation, many
            # errors (for instance whitespaces, IPv6 address litterals without
            # brackets...) are ignored.
        except ValueError:
            error(f'The provided url "{url}" could not be parsed')
979
980
981
982
983
984
985
986
987
        netloc = url_parts.netloc
    if forceIPv4:
        family = socket.AF_INET
    elif forceIPv6:
        family = socket.AF_INET6
    else:
        family = 0
    ip_set = resolved_ips(netloc, port, family, dot)

Stephane Bortzmeyer's avatar
Stephane Bortzmeyer committed
988
ok = True
989
990
for connectTo in ip_set:
    start = time.time()
991
    if dot and vhostname is not None:
992
993
994
        extracheck = vhostname
    else:
        extracheck = None
995
996
997
998
999
    if verbose and check and connectTo:
        print(f'Checking "{url}" on {connectTo} ...')
    try:
        if dot:
            conn = ConnectionDoT(url, servername=extracheck, connect=connectTo, verbose=verbose,
1000
                              debug=debug, forceIPv4=forceIPv4, forceIPv6=forceIPv6,
1001
1002
1003
                              insecure=insecure)
        else:
            conn = ConnectionDoH(url, servername=extracheck, connect=connectTo, verbose=verbose,
1004
                              debug=debug, forceIPv4=forceIPv4, forceIPv6=forceIPv6,
Alexandre's avatar