homer.py 21.4 KB
Newer Older
Stephane Bortzmeyer's avatar
Stephane Bortzmeyer committed
1
2
#!/usr/bin/env python3

3
4
5
6
7
# Homer is a DoH (DNS-over-HTTPS) and DoT (DNS-over-TLS) client. Its
# main purpose is to test DoH and DoT resolvers. Reference site is
# <https://framagit.org/bortzmeyer/homer/> See author, documentation,
# etc, there, or in the README.md included with the distribution.

Stephane Bortzmeyer's avatar
Stephane Bortzmeyer committed
8
9
10
11
12
13
# http://pycurl.io/docs/latest
import pycurl

# http://www.dnspython.org/
import dns.message

14
15
16
# https://github.com/drkjam/netaddr/
import netaddr

17
18
19
20
21
# Octobre 2019: the Python GnuTLS bindings don't work with Python 3. So we use OpenSSL.
# https://www.pyopenssl.org/
# https://pyopenssl.readthedocs.io/
import OpenSSL

Stephane Bortzmeyer's avatar
Stephane Bortzmeyer committed
22
23
24
25
26
27
import io
import sys
import base64
import getopt
import urllib.parse
import time
28
29
import socket
import ctypes
30
import re
31
import os.path
32
33
import hashlib
import base64
Stephane Bortzmeyer's avatar
Stephane Bortzmeyer committed
34

35
# Values that can be changed from the command line
36
dot = False # DoH by default
Stephane Bortzmeyer's avatar
Stephane Bortzmeyer committed
37
38
verbose = False
insecure = False
39
post = False
Stephane Bortzmeyer's avatar
Stephane Bortzmeyer committed
40
41
head = False
rtype = 'AAAA'
42
vhostname = None
Stephane Bortzmeyer's avatar
Stephane Bortzmeyer committed
43
tests = 1 # Number of repeated tests
44
ifile = None # Input file
45
delay = None
46
47
48
forceIPv4 = False
forceIPv6 = False
connectTo = None
49
50
51
# Monitoring plugin only:
host = None
path = None
Stephane Bortzmeyer's avatar
Stephane Bortzmeyer committed
52

53
54
55
# Do not change these
re_host = re.compile(r'^([0-9a-z][0-9a-z-\.]*)|([0-9:]+)|([0-9\.])$')

56
57
58
59
60
61
62
# For the monitoring plugin
STATE_OK = 0
STATE_WARNING = 1
STATE_CRITICAL = 2
STATE_UNKNOWN = 3
STATE_DEPENDENT = 4

Stephane Bortzmeyer's avatar
Stephane Bortzmeyer committed
63
64
65
def error(msg=None):
    if msg is None:
        msg = "Unknown error"
66
67
    if monitoring:
        print("%s: %s" % (url, msg))
Alexandre's avatar
Alexandre committed
68
        sys.exit(STATE_CRITICAL)
69
70
71
    else:
        print(msg,file=sys.stderr)
        sys.exit(1)
Alexandre's avatar
Alexandre committed
72

Stephane Bortzmeyer's avatar
Stephane Bortzmeyer committed
73
74
75
def usage(msg=None):
    if msg:
        print(msg,file=sys.stderr)
76
77
    print("Usage: %s [--dot] url-or-servername domain-name [DNS type]" % sys.argv[0], file=sys.stderr)
    print("See the README.md for more details.", file=sys.stderr)
Stephane Bortzmeyer's avatar
Stephane Bortzmeyer committed
78

79
def is_valid_hostname(name):
80
    name = canonicalize(name)
81
    return re_host.search(name)
82

83
84
85
86
87
88
89
90
91
def canonicalize(hostname):
    result = hostname.lower()
    # TODO handle properly the case where it fails with UnicodeError
    # (two consecutive dots for instance) to get a custom exception
    result = result.encode('idna').decode()
    if result[len(result)-1] == '.':
        result = result[:-1]
    return result

92
93
94
95
def is_valid_ip_address(addr):
    try:
        baddr = netaddr.IPAddress(addr)
    except netaddr.core.AddrFormatError:
96
97
        return (False, None)
    return (True, baddr.version)
98

99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
def is_valid_url(url):
  try:
    result = urllib.parse.urlparse(url) # A very poor validation, many
    # errors (for instance whitespaces, IPv6 address litterals without
    # brackets...) are ignored.
    return (result.scheme=="https" and result.netloc != "")
  except ValueError:
    return False

def get_certificate_san(x509cert):
    san = ""
    ext_count = x509cert.get_extension_count()
    for i in range(0, ext_count):
        ext = x509cert.get_extension(i)
        if "subjectAltName" in str(ext.get_short_name()):
            san = str(ext)
    return san

117
118
119
120
121
122
123
124
125
126
127
# Try one possible name. Names must be already canonicalized.
def match_hostname(hostname, possibleMatch):
    if possibleMatch.startswith("*."): # Wildcard
        base = possibleMatch[1:] # Skip the star
        # RFC 6125 says that we MAY accept left-most labels with
        # wildcards included (foo*bar). We don't do it here.
        try:
            (first, rest) = hostname.split(".", maxsplit=1)
        except ValueError: # One-label name
            rest = hostname
        if rest == base[1:]:
128
            return True
129
        if hostname == base[1:]:
130
            return True
131
132
133
134
135
136
137
138
        return False
    else:
        return hostname == possibleMatch

# Try all the names in the certificate
def validate_hostname(hostname, cert):
    # Complete specification is in RFC 6125. It is long and
    # complicated and I'm not sure we do it perfectly.
139
    (is_addr, family) = is_valid_ip_address(hostname)
140
    hostname = canonicalize(hostname)
141
    for alt_name in get_certificate_san(cert).split(", "):
142
        if alt_name.startswith("DNS:") and not is_addr:
143
            (start, base) = alt_name.split("DNS:")
144
            base = canonicalize(base)
145
146
            found = match_hostname(hostname, base)
            if found:
147
                return True
148
149
        elif alt_name.startswith("IP Address:") and is_addr:
            host_i = netaddr.IPAddress(hostname)
150
151
152
            (start, base) = alt_name.split("IP Address:")
            if base.endswith("\n"):
                base = base[:-1]
153
154
155
156
            try:
                base_i = netaddr.IPAddress(base)
            except netaddr.core.AddrFormatError:
                continue # Ignore broken IP addresses in certificates. Are we too liberal?
Alexandre's avatar
Alexandre committed
157
            if host_i == base_i:
158
159
                return True
        else:
160
161
162
            pass # Ignore unknown alternative name types. May be
                 # accept URI alternative names for DoH,
    # According to RFC 6125, we MUST NOT try the Common Name before the Subject Alternative Names.
163
    cn = canonicalize(cert.get_subject().commonName)
164
165
166
    found = match_hostname(hostname, cn)
    if found:
        return True
167
168
    return False

169
class Connection:
170
171
    def __init__(self, server, servername=None, connect=None, forceIPv4=False, forceIPv6=False,
                 dot=False, verbose=verbose, insecure=insecure, post=post, head=head):
172
        if dot and not is_valid_hostname(server):
173
            error("DoT requires a host name or IP address, not \"%s\"" % server)
174
175
        if not dot and not is_valid_url(url):
            error("DoH requires a valid HTTPS URL, not \"%s\"" % server)
176
        self.server = server
177
178
        self.servername = servername
        if self.servername is not None:
Alexandre's avatar
Alexandre committed
179
            self.check = self.servername
180
        else:
Alexandre's avatar
Alexandre committed
181
            self.check = self.server
182
183
184
185
        self.dot = dot
        if not self.dot:
            self.post = post
            self.head = head
186
187
        if self.dot:
            self.hasher = hashlib.sha256()
188
189
190
        self.verbose = verbose
        self.insecure = insecure
        if self.dot:
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
            if forceIPv4 and forceIPv6:
                raise Exception("Force IPv4 *or* IPv6 but not both")
            (is_addr, family) = is_valid_ip_address(self.server)
            if forceIPv4:
                if is_addr and family == 6:
                    raise Exception("You cannot force IPv4 with a litteral IPv6 address (%s)" % self.server)
                family =  socket.AF_INET
            elif forceIPv6:
                if is_addr and family == 4:
                    raise Exception("You cannot force IPv6 with a litteral IPv4 address (%s)" % self.server)
                family = socket.AF_INET6
            else:
                family = 0
            addrinfo = socket.getaddrinfo(server, 853, family)
            # May be loop over the results of getaddrinfo, to test all
            # the IP addresses? See #13.
207
            self.sock = socket.socket(addrinfo[0][0], socket.SOCK_STREAM)
208
            self.addr = addrinfo[0][4]
209
            if self.verbose:
210
                print("Connecting to %s ..." % str(self.addr))
211
212
213
214
            # With typical DoT servers, we *must* use TLS 1.2 (otherwise,
            # do_handshake fails with "OpenSSL.SSL.SysCallError: (-1, 'Unexpected
            # EOF')" Typical HTTP servers are more lax.
            self.context = OpenSSL.SSL.Context(OpenSSL.SSL.TLSv1_2_METHOD)
215
216
217
218
219
220
221
222
            if self.insecure:
                self.context.set_verify(OpenSSL.SSL.VERIFY_NONE, lambda *x: True)
            else:
                self.context.set_default_verify_paths()
                self.context.set_verify_depth(4) # Seems ignored
                self.context.set_verify(OpenSSL.SSL.VERIFY_PEER | OpenSSL.SSL.VERIFY_FAIL_IF_NO_PEER_CERT | \
                                        OpenSSL.SSL.VERIFY_CLIENT_ONCE,
                                        lambda conn, cert, errno, depth, preverify_ok: preverify_ok)
223
            self.session = OpenSSL.SSL.Connection(self.context, self.sock)
224
225
            self.session.set_tlsext_host_name(canonicalize(check).encode()) # Server Name Indication (SNI)
            self.session.connect((self.addr))
226
            # TODO We may here have exceptions such as OpenSSL.SSL.ZeroReturnError
227
228
            self.session.do_handshake()
            self.cert = self.session.get_peer_certificate()
229
230
231
            # RFC 7858, section 4.2 and appendix A
            self.publickey = self.cert.get_pubkey()
            if verbose:
232
233
234
235
                print("Certificate #%x for \"%s\", delivered by \"%s\"" % \
                      (self.cert.get_serial_number(),
                       self.cert.get_subject().commonName,
                       self.cert.get_issuer().commonName))
236
237
238
239
240
                self.hasher.update(OpenSSL.crypto.dump_publickey(OpenSSL.crypto.FILETYPE_ASN1,
                                                      self.publickey))
                self.digest = self.hasher.digest()
                print("Public key is pin-sha256=\"%s\"" % \
                      base64.standard_b64encode(self.digest).decode())
241
            if not insecure:
242
                valid = validate_hostname(check, self.cert)
243
                if not valid:
244
                    error("Certificate error: \"%s\" is not in the certificate" % (check))
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
        else: # DoH
            self.curl = pycurl.Curl()
            self.url = server
            # Does not work if pycurl was not compiled with nghttp2 (recent Debian
            # packages are OK) https://github.com/pycurl/pycurl/issues/477
            self.curl.setopt(pycurl.HTTP_VERSION, pycurl.CURL_HTTP_VERSION_2)
            self.curl.setopt(pycurl.HTTPHEADER, ["Content-type: application/dns-message"])
            if self.verbose:
                self.curl.setopt(self.curl.VERBOSE, True)
            if self.insecure:
                self.curl.setopt(pycurl.SSL_VERIFYPEER, False)   
                self.curl.setopt(pycurl.SSL_VERIFYHOST, False)
            if self.head:
                self.curl.setopt(pycurl.NOBODY, True)
            if self.post:
                 self.curl.setopt(pycurl.POST, True)
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
            if forceIPv4 and forceIPv6:
                raise Exception("Force IPv4 *or* IPv6 but not both")
            if forceIPv4:
                self.curl.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_V4)
            if forceIPv6:
                self.curl.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_V6)
            if connect is not None:
                try:
                    binaddress = netaddr.IPAddress(connect)
                    if binaddress.version == 4:
                        if forceIPv6:
                            raise Exception("You cannot force the use of IPv6 with a litteral IPv4 address (%s)" % connect)
                        repraddress = connect
                    elif binaddress.version == 6:
                        if forceIPv4:
                            raise Exception("You cannot force the use of IPv6 with a litteral IPv4 address (%s)" % connect)
                        repraddress = "[%s]" % connect
                    else:
                        raise Exception("%s is not IPv4 and not IPv6" % connect)
                except netaddr.core.AddrFormatError:
                    repraddress = connect
282
283
284
285
                try:
                    self.curl.setopt(pycurl.CONNECT_TO, ["::%s:" % repraddress,])
                except AttributeError:
                    pass # Probably an old version of libcurl, without CONNECT_TO. It appeared with 7.49.0
286
287
288
289
290
291
292
293
294
295
    def __str__(self):
        return self.server
    def end(self):
        if self.dot:
            self.session.shutdown()
            self.session.close()
        else: # DoH
            self.curl.close()
            
# Routine doing one actual test. Returns a tuple, first member is a
296
297
# result (boolean indicating success for DoT, HTTP status code for
# DoH), second member is a DNS message (or a string if there is an
298
299
# error), third member is the size of the DNS message (or None if no
# proper response).
300
def do_test(connection, qname, qtype=rtype):
Stephane Bortzmeyer's avatar
Stephane Bortzmeyer committed
301
    message = dns.message.make_query(qname, dns.rdatatype.from_text(qtype), use_edns=True, want_dnssec=True)
302
    size = None
303
    if connection.dot:
Alexandre's avatar
Alexandre committed
304
        messagew = message.to_wire()
305
        length = len(messagew)
306
        id = message.id
307
        n = connection.session.send(length.to_bytes(2, byteorder='big') + messagew)
Alexandre's avatar
Alexandre committed
308
        buf = connection.session.recv(2)
309
310
        received = int.from_bytes(buf, byteorder='big')
        buf = connection.session.recv(received)
311
312
313
        response = dns.message.from_wire(buf)
        if response.id != message.id:
            raise Exception("The ID in the answer does not match the one in the query")
314
        return (True, response, received)
315
316
317
    else: # DoH
        message.id = 0 # DoH requests that
        if connection.post:
318
            connection.curl.setopt(connection.curl.URL, connection.server)
319
320
321
322
323
324
325
326
327
328
329
            data = message.to_wire()
            connection.curl.setopt(pycurl.POSTFIELDS, data)
        else:
            dns_req = base64.urlsafe_b64encode(message.to_wire()).decode('UTF8').rstrip('=')
            connection.curl.setopt(connection.curl.URL, connection.server + ("?dns=%s" % dns_req))
        buffer = io.BytesIO()
        connection.curl.setopt(connection.curl.WRITEDATA, buffer)
        connection.curl.perform()
        rcode = connection.curl.getinfo(pycurl.RESPONSE_CODE)
        ok = True
        if rcode == 200:
330
331
332
333
            ctype = connection.curl.getinfo(pycurl.CONTENT_TYPE)
            if ctype != "application/dns-message":
                response = "Content type of the response (\"%s\") invalid" % ctype
                ok = False
334
            else:
335
336
337
338
339
340
341
342
343
344
                if not head:
                    body = buffer.getvalue()
                    try:
                        size = len(body)
                        response = dns.message.from_wire(body)
                    except dns.message.TrailingJunk: # Not DNS. Should
                        # not happen for a content type
                        # application/dns-message but who knows?
                        response = "ERROR Not proper DNS data, trailing junk \"%s\"" % body
                        ok = False
Alexandre's avatar
Alexandre committed
345
                    except dns.name.BadLabelType: # Not DNS.
346
347
348
349
                        response = "ERROR Not proper DNS data (wrong path in the URL?) \"%s\"" % body[:100]
                        ok = False
                else:
                    response = "HEAD successful"
350
        else:
351
            ok = False
352
353
            body =  buffer.getvalue()
            if len(body) == 0:
354
355
356
357
                response = "[No details]"
            else:
                response = body
            buffer.close()
358
        return (rcode, response, size)
Alexandre's avatar
Alexandre committed
359

360
# Main program
361
362
363
364
365
366
me = os.path.basename(sys.argv[0])
monitoring = (me == "check_doh" or me == "check_dot")
if not monitoring:
    name = None
    message = None
    try:
367
368
369
        optlist, args = getopt.getopt (sys.argv[1:], "hvPkeV:r:f:d:t46",
                                       ["help", "verbose", "dot", "head", "insecure", "POST", "vhost=",
                                        "repeat=", "file=", "delay=", "v4only", "v6only"])
370
371
372
373
        for option, value in optlist:
            if option == "--help" or option == "-h":
                usage()
                sys.exit(0)
374
375
            elif option == "--dot" or option == "-t":
                dot = True
376
377
            elif option == "--verbose" or option == "-v":
                verbose = True
378
            elif option == "--HEAD" or option == "--head" or option == "-e":
379
                head = True
380
            elif option == "--POST" or option == "--post" or option == "-P":
381
                post = True
382
383
            elif option == "--vhost" or option == "-V":
                vhostname = value
384
385
            elif option == "--insecure" or option == "-k":
                insecure = True
386
387
388
389
390
391
392
393
394
395
            elif option == "--repeat" or option == "-r":
                tests = int(value)
                if tests <= 1:
                    error("--repeat needs a value > 1")
            elif option == "--delay" or option == "-d":
                delay = float(value)
                if delay <= 0:
                    error("--delay needs a value > 0")
            elif option == "--file" or option == "-f":
                ifile = value
396
397
398
399
            elif option == "-4" or option == "v4only":
                forceIPv4 = True
            elif option == "-6" or option == "v6only":
                forceIPv6 = True
400
401
402
403
404
405
406
407
408
409
410
411
            else:
                error("Unknown option %s" % option)
    except getopt.error as reason:
        usage(reason)
        sys.exit(1)
    if tests <= 1 and delay is not None:
        error("--delay makes no sense if there is no repetition")
    if post and head:
        usage("POST or HEAD but not both")
        sys.exit(1)
    if dot and (post or head):
        usage("POST or HEAD makes non sense for DoT")
Alexandre's avatar
Alexandre committed
412
        sys.exit(1)
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
    if ifile is None and (len(args) != 2 and len(args) != 3):
        usage("Wrong number of arguments")
        sys.exit(1)
    if ifile is not None and len(args) != 1:
        usage("Wrong number of arguments (if --file is used, do not indicate the domain name)")
        sys.exit(1)
    url = args[0]
    if ifile is None:
        name = args[1]
        if len(args) == 3:
            rtype = args[2]
else: # Monitoring plugin
    dot = (me == "check_dot")
    name = None
    try:
428
        optlist, args = getopt.getopt (sys.argv[1:], "H:n:p:V:t:Pih46")
429
430
431
432
        for option, value in optlist:
            if option == "-H":
                host = value
            elif option == "-V":
Alexandre's avatar
Alexandre committed
433
                vhostname = value
434
435
436
437
438
439
440
441
442
443
            elif option == "-n":
                name = value
            elif option == "-t":
                rtype = value
            elif option == "-p":
                path = value
            elif option == "-P":
                post = True
            elif option == "-h":
                head = True
444
445
            elif option == "-i":
                insecure = True
446
447
448
449
            elif option == "-4":
                forceIPv4 = True
            elif option == "-6":
                forceIPv6 = True
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
            else:
                # Should never occur, it is trapped by getopt
                print("Unknown option %s" % option)
                sys.exit(STATE_UNKNOWN)
    except getopt.error as reason:
        print("Option parsing problem %s" % reason)
        sys.exit(STATE_UNKNOWN)
    if len(args) > 0:
        print("Too many arguments (\"%s\")" % args)
        sys.exit(STATE_UNKNOWN)
    if host is None or name is None:
        print("Host (-H) and name to lookup (-n) are necessary")
        sys.exit(STATE_UNKNOWN)
    if post and head:
        print("POST or HEAD but not both")
        sys.exit(STATE_UNKNOWN)
466
467
468
469
470
471
    if dot and (post or head):
        print("POST or HEAD makes no sense for DoT")
        sys.exit(STATE_UNKNOWN)
    if dot and path:
        print("URL path makes no sense for DoT")
        sys.exit(STATE_UNKNOWN)
472
473
474
    if dot:
        url = host
    else:
475
476
        if vhostname is None or vhostname == host:
            connectTo = None
477
            url = "https://%s/" % host
Stephane Bortzmeyer's avatar
Stephane Bortzmeyer committed
478
        else:
479
            connectTo = host
Alexandre's avatar
Alexandre committed
480
            url = "https://%s/" % vhostname
481
482
483
484
        if path is not None:
            if path.startswith("/"):
                path = path[1:]
            url += path
Stephane Bortzmeyer's avatar
Stephane Bortzmeyer committed
485
ok = True
Alexandre's avatar
Alexandre committed
486
start = time.time()
487
try:
488
    if dot and vhostname is not None:
489
490
491
        extracheck = vhostname
    else:
        extracheck = None
492
493
494
    conn = Connection(url, dot=dot, servername=extracheck, connect=connectTo, verbose=verbose,
                      forceIPv4=forceIPv4, forceIPv6=forceIPv6,
                      insecure=insecure, post=post, head=head)
495
496
except TimeoutError:
    error("timeout")
497
498
except ConnectionRefusedError:
    error("Connection to server refused")
499
500
if ifile is not None:
    input = open(ifile)
Stephane Bortzmeyer's avatar
Stephane Bortzmeyer committed
501
for i in range (0, tests):
502
503
504
505
506
507
508
509
510
511
512
    if tests > 1:
        print("\nTest %i" % i)
    if ifile is not None:
        line = input.readline()
        if line[:-1] == "":
            error("Not enough data in %s for the %i tests" % (ifile, tests))
        if line.find(' ') == -1:
            name = line[:-1]
            rtype = 'AAAA'
        else:
            (name, rtype) = line.split()
513
    (rcode, msg, size) = do_test(conn, name, rtype)
514
    if (dot and rcode) or (not dot and rcode == 200):
515
516
517
518
519
520
521
522
        if not monitoring:
            print(msg)
        else:
            if size is not None and size > 0:
                print("%s OK - %s" % (url, "No error for %s/%s, %i bytes received" % (name, rtype, size)))
            else:
                print("%s OK - %s" % (url, "No error"))
            sys.exit(STATE_OK)
Stephane Bortzmeyer's avatar
Stephane Bortzmeyer committed
523
    else:
524
525
526
527
        if not monitoring:
            if dot:
                print("Error: %s" % msg, file=sys.stderr)
            else:
528
529
               try:
                   msg = msg.decode()
530
               except (UnicodeDecodeError, AttributeError):
531
532
                   pass # Sometimes, msg can be binary, or Latin-1
               print("HTTP error %i: %s" % (rcode, msg), file=sys.stderr)
533
        else:
534
            if not dot:
535
                print("%s HTTP error - %i: %s" % (url, rcode, msg))
536
537
538
            else:
                print("%s Error - %i: %s" % (url, rcode, msg))
            sys.exit(STATE_CRITICAL)
Stephane Bortzmeyer's avatar
Stephane Bortzmeyer committed
539
        ok = False
540
541
    if tests > 1 and i == 0:
        start2 = time.time()
542
543
    if delay is not None:
        time.sleep(delay)
Stephane Bortzmeyer's avatar
Stephane Bortzmeyer committed
544
stop = time.time()
545
546
547
548
if tests > 1:
    extra = ", %.2f ms/request if we ignore the first one" % ((stop-start2)*1000/(tests-1))
else:
    extra = ""
549
550
if not monitoring:
    print("\nTotal elapsed time: %.2f seconds (%.2f ms/request %s)" % (stop-start, (stop-start)*1000/tests, extra))
551
552
if ifile is not None:
    input.close()
553
conn.end()
Stephane Bortzmeyer's avatar
Stephane Bortzmeyer committed
554
if ok:
555
556
557
558
    if not monitoring:
        sys.exit(0)
    else:
        sys.exit(STATE_OK)
Stephane Bortzmeyer's avatar
Stephane Bortzmeyer committed
559
else:
560
561
562
563
    if not monitoring:
        sys.exit(1)
    else:
        sys.exit(STATE_CRITICAL)
Alexandre's avatar
Alexandre committed
564