remoh.py 27.9 KB
Newer Older
Stephane Bortzmeyer's avatar
Stephane Bortzmeyer committed
1
2
#!/usr/bin/env python3

Alexandre's avatar
Alexandre committed
3
# Remoh is a DoH (DNS-over-HTTPS) and DoT (DNS-over-TLS) client. Its
4
5
6
7
# main purpose is to test DoH and DoT resolvers. Reference site is
# <https://framagit.org/bortzmeyer/homer/> See author, documentation,
# etc, there, or in the README.md included with the distribution.

Stephane Bortzmeyer's avatar
Stephane Bortzmeyer committed
8
9
10
11
import sys
import getopt
import urllib.parse
import time
12
import socket
Alexandre's avatar
Alexandre committed
13
import dns
Stephane Bortzmeyer's avatar
Stephane Bortzmeyer committed
14

Alexandre's avatar
Alexandre committed
15
16
17
18
19
20
21
22
23
24
25
26
27
try:
    # http://pycurl.io/docs/latest
    import pycurl

    # Octobre 2019: the Python GnuTLS bindings don't work with Python 3. So we use OpenSSL.
    # https://www.pyopenssl.org/
    # https://pyopenssl.readthedocs.io/
    import OpenSSL
except ImportError as e:
    print("Error: missing module")
    print(e)
    sys.exit(1)

Alexandre's avatar
Alexandre committed
28
import remoh
Alexandre's avatar
Alexandre committed
29

30
# Values that can be changed from the command line
Alexandre's avatar
Alexandre committed
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
class opts:
    dot = False # DoH by default
    verbose = False
    debug = False
    insecure = False
    post = False
    head = False
    dnssec = False
    edns = True
    no_ecs = True
    sni = True
    rtype = 'AAAA'
    vhostname = None
    tests = 1 # Number of repeated tests
    key = None # SPKI
    ifile = None # Input file
    delay = None
    forceIPv4 = False
    forceIPv6 = False
    connectTo = None
    pipelining = False
    max_in_flight = 20
    multistreams = False
    display_results = True
    show_time = False
    check = False
    mandatory_level = None
    check_additional = True
59

60
def error(msg=None, exit=True):
Stephane Bortzmeyer's avatar
Stephane Bortzmeyer committed
61
62
    if msg is None:
        msg = "Unknown error"
63
    print(msg, file=sys.stderr)
Alexandre's avatar
Alexandre committed
64
65
66
67

def error_and_exit(msg=None):
    error(msg)
    sys.exit(1)
Alexandre's avatar
Alexandre committed
68

Stephane Bortzmeyer's avatar
Stephane Bortzmeyer committed
69
70
71
def usage(msg=None):
    if msg:
        print(msg,file=sys.stderr)
Alexandre's avatar
Alexandre committed
72
    print("Usage: %s [options] url-or-servername domain-name [DNS type]" % sys.argv[0], file=sys.stderr)
Alexandre's avatar
Alexandre committed
73
74
    print("""Options
    -t --dot            Use DoT (by default use DoH)
75
76
77
78
79
    -k --insecure       Do not check the certificate
    -4 --v4only         Force IPv4 resolution of url-or-servername
    -6 --v6only         Force IPv6 resolution of url-or-servername
    -v --verbose        Make the program more talkative
    --debug             Make the program even more talkative than -v
Alexandre's avatar
Alexandre committed
80
    -r --repeat <N>     Perform N times the query. If used with -f, read up to
81
                        <N> lines of the <file>
Alexandre's avatar
Alexandre committed
82
83
84
    -d --delay <T>      Time to wait in seconds between each synchronous
                        request (only with --repeat)
    -f --file <file>    Read domain names from <file>, one per row with an
85
86
87
                        optional DNS type. Read the first line only, use
                        --repeat N to read up to N lines of the file
    --check             Perform a set of predefined tests
Alexandre's avatar
Alexandre committed
88
89
90
91
92
    --mandatory-level <level>
                        Define the <level> of test to perform (only with
                        --check)
                        Available <level> : legal, necessary, nicetohave
    --no-display-results
93
                        Disable output of DNS response
Alexandre's avatar
Alexandre committed
94
95
96
97
    --dnssec            Request DNSSEC data (signatures)
    --noedns            Disable EDNS, default is to indicate EDNS support
    --ecs               Send ECS to authoritative servers, default is to
                        refuse it
Alexandre's avatar
Alexandre committed
98
    -V --vhost <vhost>  Use a specific virtual host
Alexandre's avatar
Alexandre committed
99
100
    -h --help           Print this message

101
102
103
104
105
106
107
108
109
110
111
  DoH only options:
    -P --post --POST    Use HTTP POST method for all the transfers
    -e --head --HEAD    Use HTTP HEAD method for all the transfers
    --multistreams      Use HTTP/2 streams, needs an input file with -f
    --time              Display the time elapsed for the query (only with
                        --multistreams)

  DoT only options:
    --key <key>         Authenticate a DoT resolver with its public <key> in
                        base64
    --nosni             Do not perform SNI
Alexandre's avatar
Alexandre committed
112
113
114
    --pipelining        Pipeline the requests, needs an input file with -f
    --max-in-flight <M> Maximum number of concurrent requests in parallel (only
                        with --pipelining)
115

Alexandre's avatar
Alexandre committed
116
117
118
119
120
    url-or-servername   The URL or domain name of the DoT/DoH server
    domain-name         The domain name to resolve, not required if -f is
                        provided
    DNS type            The DNS record type to resolve, default AAAA
    """, file=sys.stderr)
121
    print("See the README.md for more details.", file=sys.stderr)
Stephane Bortzmeyer's avatar
Stephane Bortzmeyer committed
122

Alexandre's avatar
Alexandre committed
123
124
125
126
def get_next_domain(input_file):
    name, rtype = 'framagit.org', 'AAAA'
    line = input_file.readline()
    if line[:-1] == "":
Alexandre's avatar
Alexandre committed
127
        error("Not enough data in %s for the %i tests" % (opts.ifile, opts.tests))
Alexandre's avatar
Alexandre committed
128
129
130
131
132
133
134
    if line.find(' ') == -1:
        name = line[:-1]
        rtype = 'AAAA'
    else:
        (name, rtype) = line.split()
    return name, rtype

Alexandre's avatar
Alexandre committed
135
136
137
138
139
140
141
142
143
144
145
def print_info(msg, ip=None, prefix=None, msg_type=None, fd=sys.stdout):
    output = ""
    if ip:
        output += '%s: ' % ip
    if prefix:
        output += '%s: ' % prefix
    if msg_type:
        output += '%s: ' % msg_type
    output += '%s' % msg
    print(output, file=fd)

146
def print_result(connection, request, prefix=None, display_err=True):
147
148
149
150
151
152
    dot = connection.dot
    server = connection.server
    rcode = request.rcode
    msg = request.response
    size = request.response_size
    if (dot and rcode) or (not dot and rcode == 200):
153
154
155
156
        if not opts.dot and opts.show_time:
            connection.print_time(connection.curl_handle)
        if opts.display_results and (not opts.check or opts.verbose):
            print(msg)
157
    else:
158
        if display_err:
Alexandre's avatar
Alexandre committed
159
            ip = connection.connect_to
160
            if dot:
Alexandre's avatar
Alexandre committed
161
                msg_type = 'Error'
162
            else:
163
164
165
166
               try:
                   msg = msg.decode()
               except (UnicodeDecodeError, AttributeError):
                   pass # Sometimes, msg can be binary, or Latin-1
Alexandre's avatar
Alexandre committed
167
168
               msg_type = 'HTTP error %i' % rcode
            print_info(msg, ip, prefix, msg_type, fd=sys.stderr)
169

Alexandre's avatar
Alexandre committed
170
171
172
173
174
175
176
177
178
179
180
181
182
183

def print_check_result(test_name, ok, verbose=True):
    if verbose:
        print(test_name, end=' : ')
        if ok:
            print('OK')
        else:
            print('KO')

def check_dot_two_requests(connection, opts):
    # not using a DoT connection -> exit the test
    if not connection.dot:
        return True

Alexandre's avatar
Alexandre committed
184
185
    r1 = remoh.RequestDOT('framagit.org', qtype=opts.rtype, use_edns=opts.edns, want_dnssec=opts.dnssec, no_ecs=opts.no_ecs)
    r2 = remoh.RequestDOT('afnic.fr', qtype=opts.rtype, use_edns=opts.edns, want_dnssec=opts.dnssec, no_ecs=opts.no_ecs)
Alexandre's avatar
Alexandre committed
186
187

    requests = []
Alexandre's avatar
Alexandre committed
188
    requests.append(('Test 1', r1, remoh.mandatory_levels["legal"]))
Alexandre's avatar
Alexandre committed
189
    # RFC 7858 section 3.3, SHOULD accept several requests on one connection.
Alexandre's avatar
Alexandre committed
190
    requests.append(('Test 2', r2, remoh.mandatory_levels["necessary"]))
Alexandre's avatar
Alexandre committed
191
192
193
194
195
196
197
198

    return do_check(connection, requests, opts)

def check_doh_methods(connection, opts):
    # not using a DoH connection -> exit the test
    if connection.dot:
        return True

Alexandre's avatar
Alexandre committed
199
200
    r1 = remoh.RequestDOH('framagit.org', qtype=opts.rtype, use_edns=opts.edns, want_dnssec=opts.dnssec, no_ecs=opts.no_ecs)
    r2 = remoh.RequestDOH('afnic.fr', qtype=opts.rtype, use_edns=opts.edns, want_dnssec=opts.dnssec, no_ecs=opts.no_ecs)
Alexandre's avatar
Alexandre committed
201
    r2.post = True
Alexandre's avatar
Alexandre committed
202
    r3 = remoh.RequestDOH('www.rfc-editor.org', qtype=opts.rtype, use_edns=opts.edns, want_dnssec=opts.dnssec, no_ecs=opts.no_ecs)
Alexandre's avatar
Alexandre committed
203
204
205
    r3.head = True

    requests = []
Alexandre's avatar
Alexandre committed
206
207
    requests.append(('Test GET', r1, remoh.mandatory_levels["legal"])) # RFC 8484, section 4.1
    requests.append(('Test POST', r2, remoh.mandatory_levels["legal"])) # RFC 8484, section 4.1
Alexandre's avatar
Alexandre committed
208
    # HEAD method is not mentioned in RFC 8484 (see section 4.1), so just "nice to have".
Alexandre's avatar
Alexandre committed
209
    requests.append(('Test HEAD', r3, remoh.mandatory_levels["nicetohave"]))
Alexandre's avatar
Alexandre committed
210
211
212

    return do_check(connection, requests, opts)

Alexandre's avatar
Alexandre committed
213
def check_doh_header(connection, opts, level=remoh.mandatory_levels["nicetohave"],
Alexandre's avatar
Alexandre committed
214
215
216
217
218
219
220
221
222
223
224
        accept="application/dns-message", content_type="application/dns-message"):
    # change the MIME value and see what happens
    # based on the RFC only application/dns-message must be supported, any
    # other MIME type can be also supported, but nothing is said on that

    # not using a DoH connection -> exit the test
    if connection.dot:
        return True

    header = ["Accept: %s" % accept, "Content-type: %s" % content_type]
    test_name = "Test Header MIME: %s " % ", ".join(h for h in header)
Alexandre's avatar
Alexandre committed
225
    r1 = remoh.RequestDOH('curl.haxx.se', qtype=opts.rtype, use_edns=opts.edns, want_dnssec=opts.dnssec, no_ecs=opts.no_ecs)
Alexandre's avatar
Alexandre committed
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
    r1.post = True

    requests = []
    requests.append((test_name, r1, level))

    handle = connection.curl_handle
    handle.setopt(pycurl.HTTPHEADER, header)

    ok = do_check(connection, requests, opts)

    default_accept = "application/dns-message"
    default_ct = "application/dns-message"
    default_header = ["Accept: %s" % default_accept, "Content-type: %s" % default_ct]
    handle.setopt(pycurl.HTTPHEADER, default_header)

    return ok

def do_check(connection, requests, opts):
Alexandre's avatar
Alexandre committed
244
    ok = True
245
    for request_pack in requests:
Alexandre's avatar
Alexandre committed
246
247
248
249
250
251
252
253
254
        test_name, request, level = request_pack

        # the test level is too small, therefore shouldn't be run
        if level < opts.mandatory_level:
            continue

        request.to_wire()

        if connection.debug:
255
            print(test_name)
Alexandre's avatar
Alexandre committed
256
        if connection.dot:
Alexandre's avatar
Alexandre committed
257
258
            bundle = request
        else:
259
260
261
            handle = connection.curl_handle
            handle.prepare(handle, connection, request)
            bundle = handle
Alexandre's avatar
Alexandre committed
262

Alexandre's avatar
Alexandre committed
263
264
        try:
            connection.send_and_receive(bundle)
Alexandre's avatar
Alexandre committed
265
        except (remoh.ConnectionException, remoh.DOHException) as e:
Alexandre's avatar
Alexandre committed
266
267
            ok = False
            print_check_result(test_name, ok, verbose=connection.verbose)
Alexandre's avatar
Alexandre committed
268
            print_info(e, connection.connect_to, fd=sys.stderr)
Alexandre's avatar
Alexandre committed
269
270
            continue

271
272
        if level >= opts.mandatory_level:
            ok = request.check_response(connection.debug)
273
274
            if request.rcode == 415 and 'Test Header MIME' in test_name:
                ok = True
Alexandre's avatar
Alexandre committed
275
        print_check_result(test_name, ok, verbose=connection.verbose)
276
277
278
        print_result(connection, request, prefix=test_name, display_err=not ok)
        if not ok:
            break
Alexandre's avatar
Alexandre committed
279
    return ok
280

Alexandre's avatar
Alexandre committed
281
def check_truncated_query(connection, opts, level=remoh.mandatory_levels["nicetohave"]):
Alexandre's avatar
Alexandre committed
282
283
284
285
    # send truncated DNS request to the server and expect a HTTP return code
    # either equal to 200 or in the 400 range
    # in case the server answers with 200, look for a FORMERR error in the DNS
    # response
Alexandre's avatar
Alexandre committed
286
287
288
289
290

    # the test level is too small, therefore shouldn't be run
    if level < opts.mandatory_level:
       return True

291
    ok = True
Alexandre's avatar
Alexandre committed
292

293
    test_name = 'Test truncated data'
Alexandre's avatar
Alexandre committed
294
295

    if connection.dot:
Alexandre's avatar
Alexandre committed
296
        request = remoh.RequestDOT('example.com', qtype=opts.rtype, use_edns=opts.edns, want_dnssec=opts.dnssec, no_ecs=opts.no_ecs)
Alexandre's avatar
Alexandre committed
297
    else:
Alexandre's avatar
Alexandre committed
298
        request = remoh.RequestDOH('example.com', qtype=opts.rtype, use_edns=opts.edns, want_dnssec=opts.dnssec, no_ecs=opts.no_ecs)
Alexandre's avatar
Alexandre committed
299
300
301
302
303
        request.post = True

    request.trunc_data()

    if connection.debug:
304
        print(test_name)
Alexandre's avatar
Alexandre committed
305
    if connection.dot:
306
        bundle = request
307
    else:
308
309
310
        handle = connection.curl_handle
        handle.prepare(handle, connection, request)
        bundle = handle
Alexandre's avatar
Alexandre committed
311

312
    try:
313
        # 8.8.8.8 replies FORMERR but most DoT servers violently shut down the connection (which is legal)
Alexandre's avatar
Alexandre committed
314
        connection.send_and_receive(bundle, dump=connection.debug)
315
    except OpenSSL.SSL.ZeroReturnError: # This is acceptable
Alexandre's avatar
Alexandre committed
316
        return True
Alexandre's avatar
Alexandre committed
317
    except dns.exception.FormError as e: # This is also acceptable
318
319
320
        # Some DSN resolvers will echo mangled requests with
        # the RCODE set to FORMERR
        # so response can not be parsed in this case
Alexandre's avatar
Alexandre committed
321
        print_info(e, connection.connect_to, test_name, 'Info', fd=sys.stderr)
Alexandre's avatar
Alexandre committed
322
        return True
Alexandre's avatar
Alexandre committed
323
    except remoh.ConnectionDOTException as e:
Alexandre's avatar
Alexandre committed
324
        print_info(e, connection.connect_to, test_name, 'Info', fd=sys.stderr)
Alexandre's avatar
Alexandre committed
325
        return connection.state == 'CONN_CLOSED'
Alexandre's avatar
Alexandre committed
326
    except remoh.DOHException as e:
Alexandre's avatar
Alexandre committed
327
        print_info(e, connection.connect_to, test_name, 'Info', fd=sys.stderr)
Alexandre's avatar
Alexandre committed
328
        return False
Alexandre's avatar
Alexandre committed
329

Alexandre's avatar
Alexandre committed
330
331
    if request.check_response(connection.debug): # FORMERR is expected
        if connection.dot:
Alexandre's avatar
Alexandre committed
332
333
334
            ok = request.rcode == dns.rcode.FORMERR
        else:
            ok = (request.response.rcode() == dns.rcode.FORMERR)
335
    else:
Alexandre's avatar
Alexandre committed
336
        if connection.dot:
Alexandre's avatar
Alexandre committed
337
            ok = False
Alexandre's avatar
Alexandre committed
338
339
340
341
342
        else: # only a 400 range HTTP code is acceptable
              # if we send garbage to the server, it seems reasonable that it
              # does not fail, which means we don't accept a 500 range HTTP
              # error code (even so it means the server failed to process the
              # input data)
Alexandre's avatar
Alexandre committed
343
            ok = (request.rcode >= 400 and request.rcode < 500)
Alexandre's avatar
Alexandre committed
344
345

    print_check_result(test_name, ok, verbose=connection.verbose)
Alexandre's avatar
Alexandre committed
346
    print_result(connection, request, prefix=test_name, display_err=not ok)
347

Alexandre's avatar
Alexandre committed
348
    return ok
349
350

def run_check(connection):
Alexandre's avatar
Alexandre committed
351
352
353
354
355
    ok = True
    if connection.dot:
        ok = check_dot_two_requests(connection, opts)
    else:
        ok = check_doh_methods(connection, opts)
Alexandre's avatar
Alexandre committed
356
    if not ok and opts.mandatory_level >= remoh.mandatory_levels["nicetohave"]:
357
        return False
Alexandre's avatar
Alexandre committed
358

Alexandre's avatar
Alexandre committed
359
360
361
    # TODO we miss the tests of pipelining and out-of-order for DoT and
    # multistreams for DoH

Alexandre's avatar
Alexandre committed
362
363
364
365
366
    # Test that different Header values are not breaking anything
    if not connection.dot:
        # The DoH server is right to reject these (Example: 'HTTP
        # error 415: only Content-Type: application/dns-message is
        # supported')
Alexandre's avatar
Alexandre committed
367
368
        ok = check_doh_header(connection, opts, level=remoh.mandatory_levels["nocrash"], accept="text/html") and ok
        ok = check_doh_header(connection, opts, level=remoh.mandatory_levels["nocrash"], content_type="text/html") and ok
Alexandre's avatar
Alexandre committed
369
370

    # test if a truncated query breaks anything
Alexandre's avatar
Alexandre committed
371
    ok = check_truncated_query(connection, opts, level=remoh.mandatory_levels["nocrash"]) and ok
Alexandre's avatar
Alexandre committed
372
373

    return ok
374

Alexandre's avatar
Alexandre committed
375
def resolved_ips(host, port, family, dot=False):
376
377
378
    try:
        addr_list = socket.getaddrinfo(host, port, family)
    except socket.gaierror:
Alexandre's avatar
Alexandre committed
379
        error_and_exit("Could not resolve \"%s\"" % host)
380
381
    ip_set = { addr[4][0] for addr in addr_list }
    return ip_set
Alexandre's avatar
Alexandre committed
382

383
def parse_opts(opts):
384
    name = None
385
386
    rtype = opts.rtype

387
    try:
388
        optlist, args = getopt.getopt (sys.argv[1:], "hvPkeV:r:f:d:t46",
389
                                       ["help", "verbose", "debug", "dot",
Alexandre's avatar
Alexandre committed
390
391
                                        "head", "HEAD", "post", "POST",
                                        "insecure", "vhost=", "multistreams",
392
                                        "pipelining", "max-in-flight=", "key=",
393
                                        "dnssec", "noedns", "ecs", "nosni",
394
                                        "no-display-results", "time",
395
                                        "file=", "repeat=", "delay=",
396
                                        "v4only", "v6only",
397
                                        "check", "mandatory-level="])
398
399
400
401
        for option, value in optlist:
            if option == "--help" or option == "-h":
                usage()
                sys.exit(0)
402
            elif option == "--dot" or option == "-t":
Alexandre's avatar
Alexandre committed
403
                opts.dot = True
404
            elif option == "--verbose" or option == "-v":
Alexandre's avatar
Alexandre committed
405
                opts.verbose = True
Alexandre's avatar
Alexandre committed
406
            elif option == "--debug":
Alexandre's avatar
Alexandre committed
407
408
                opts.debug = True
                opts.verbose = True
409
            elif option == "--HEAD" or option == "--head" or option == "-e":
Alexandre's avatar
Alexandre committed
410
                opts.head = True
411
            elif option == "--POST" or option == "--post" or option == "-P":
Alexandre's avatar
Alexandre committed
412
                opts.post = True
413
            elif option == "--vhost" or option == "-V":
Alexandre's avatar
Alexandre committed
414
                opts.vhostname = value
415
            elif option == "--insecure" or option == "-k":
Alexandre's avatar
Alexandre committed
416
                opts.insecure = True
417
            elif option == "--multistreams":
Alexandre's avatar
Alexandre committed
418
                opts.multistreams = True
419
            elif option == "--no-display-results":
Alexandre's avatar
Alexandre committed
420
                opts.display_results = False
421
            elif option == "--time":
Alexandre's avatar
Alexandre committed
422
                opts.show_time = True
423
            elif option == "--dnssec":
Alexandre's avatar
Alexandre committed
424
                opts.dnssec = True
425
            elif option == "--nosni":
Alexandre's avatar
Alexandre committed
426
                opts.sni = False
427
428
429
430
            elif option == "--noedns": # Warning: it will mean the
                                       # resolver may send ECS
                                       # information to the
                                       # authoritative name servers.
Alexandre's avatar
Alexandre committed
431
                opts.edns = False
432
            elif option == "--ecs":
Alexandre's avatar
Alexandre committed
433
                opts.no_ecs = False
434
            elif option == "--repeat" or option == "-r":
Alexandre's avatar
Alexandre committed
435
436
                opts.tests = int(value)
                if opts.tests <= 1:
Alexandre's avatar
Alexandre committed
437
                    error_and_exit("--repeat needs a value > 1")
438
            elif option == "--delay" or option == "-d":
Alexandre's avatar
Alexandre committed
439
                opts.delay = float(value)
440
                if opts.delay <= 0:
Alexandre's avatar
Alexandre committed
441
                    error_and_exit("--delay needs a value > 0")
442
            elif option == "--file" or option == "-f":
Alexandre's avatar
Alexandre committed
443
                opts.ifile = value
444
            elif option == "--key":
Alexandre's avatar
Alexandre committed
445
                opts.key = value
446
            elif option == "-4" or option == "--v4only":
Alexandre's avatar
Alexandre committed
447
                opts.forceIPv4 = True
448
            elif option == "-6" or option == "--v6only":
Alexandre's avatar
Alexandre committed
449
                opts.forceIPv6 = True
450
            elif option == "--pipelining":
Alexandre's avatar
Alexandre committed
451
                opts.pipelining = True
452
            elif option == "--max-in-flight":
Alexandre's avatar
Alexandre committed
453
                opts.max_in_flight = int(value)
454
                if opts.max_in_flight <= 0:
Alexandre's avatar
Alexandre committed
455
                    error_and_exit("--max_in_flight but be > 0")
456
                if opts.max_in_flight >= 65536:
Alexandre's avatar
Alexandre committed
457
                    error_and_exit("Because of a limit of the DNS protocol (the size of the query ID) --max_in_flight must be < 65 536")
458
            elif option == "--check":
Alexandre's avatar
Alexandre committed
459
460
                opts.check = True
                opts.display_results = False
461
            elif option == "--mandatory-level":
Alexandre's avatar
Alexandre committed
462
                opts.mandatory_level = value
463
            else:
Alexandre's avatar
Alexandre committed
464
                error_and_exit("Unknown option %s" % option)
Alexandre's avatar
Alexandre committed
465
    except (getopt.error, ValueError) as reason:
Alexandre's avatar
Alexandre committed
466
        error_and_exit(reason)
467

Alexandre's avatar
Alexandre committed
468
    if opts.delay is not None and opts.multistreams:
Alexandre's avatar
Alexandre committed
469
        error_and_exit("--delay makes no sense with multistreams")
Alexandre's avatar
Alexandre committed
470
    if opts.tests <= 1 and opts.delay is not None:
Alexandre's avatar
Alexandre committed
471
        error_and_exit("--delay makes no sense if there is no repetition")
Alexandre's avatar
Alexandre committed
472
    if not opts.dot and opts.pipelining:
Alexandre's avatar
Alexandre committed
473
        error_and_exit("Pipelining is only accepted for DoT")
Alexandre's avatar
Alexandre committed
474
    if opts.dot and (opts.post or opts.head):
Alexandre's avatar
Alexandre committed
475
        error_and_exit("POST or HEAD makes non sense for DoT")
Alexandre's avatar
Alexandre committed
476
    if opts.post and opts.head:
Alexandre's avatar
Alexandre committed
477
        error_and_exit("POST or HEAD but not both")
Alexandre's avatar
Alexandre committed
478
    if opts.pipelining and opts.ifile is None:
Alexandre's avatar
Alexandre committed
479
        error_and_exit("Pipelining requires an input file")
Alexandre's avatar
Alexandre committed
480
    if opts.check and opts.multistreams:
Alexandre's avatar
Alexandre committed
481
        error_and_exit("--check and --multistreams are not compatible")
Alexandre's avatar
Alexandre committed
482
    if opts.dot and opts.multistreams:
Alexandre's avatar
Alexandre committed
483
        error_and_exit("Multi-streams makes no sense for DoT")
Alexandre's avatar
Alexandre committed
484
    if opts.multistreams and opts.ifile is None:
Alexandre's avatar
Alexandre committed
485
        error_and_exit("Multi-streams requires an input file")
Alexandre's avatar
Alexandre committed
486
    if opts.show_time and opts.dot:
Alexandre's avatar
Alexandre committed
487
        error_and_exit("--time cannot be used with --dot")
Alexandre's avatar
Alexandre committed
488
    if not opts.edns and not opts.no_ecs:
Alexandre's avatar
Alexandre committed
489
        error_and_exit("ECS requires EDNS")
Alexandre's avatar
Alexandre committed
490
    if opts.mandatory_level is not None and \
Alexandre's avatar
Alexandre committed
491
       opts.mandatory_level not in remoh.mandatory_levels.keys():
Alexandre's avatar
Alexandre committed
492
        error_and_exit("Unknown mandatory level \"%s\"" % opts.mandatory_level)
Alexandre's avatar
Alexandre committed
493
    if opts.mandatory_level is not None and not opts.check:
Alexandre's avatar
Alexandre committed
494
        error_and_exit("--mandatory-level only makes sense with --check")
Alexandre's avatar
Alexandre committed
495
496
    if opts.mandatory_level is None:
        opts.mandatory_level = "necessary"
Alexandre's avatar
Alexandre committed
497
    opts.mandatory_level = remoh.mandatory_levels[opts.mandatory_level]
Alexandre's avatar
Alexandre committed
498
    if opts.ifile is None and (len(args) != 2 and len(args) != 3):
Alexandre's avatar
Alexandre committed
499
        error_and_exit("Wrong number of arguments")
Alexandre's avatar
Alexandre committed
500
    if opts.ifile is not None and len(args) != 1:
Alexandre's avatar
Alexandre committed
501
        error_and_exit("Wrong number of arguments (if --file is used, do not indicate the domain name)")
502
    url = args[0]
Alexandre's avatar
Alexandre committed
503
    if opts.ifile is None:
504
505
        name = args[1]
        if len(args) == 3:
506
            opts.rtype = args[2]
507

508
    return (url, name)
509

Alexandre's avatar
Alexandre committed
510
511
def run_default(name, connection, opts):
    ok = True
512
    start = time.time()
513

Alexandre's avatar
Alexandre committed
514
515
    if opts.multistreams:
        connection.init_multi()
516

Alexandre's avatar
Alexandre committed
517
    for i in range (0, opts.tests):
Alexandre's avatar
Alexandre committed
518
519
520
521

        if not opts.pipelining and not opts.multistreams:
            if opts.tests > 1 and (opts.verbose or opts.display_results):
                print("\nTest %i" % i)
522

Alexandre's avatar
Alexandre committed
523
524
        if opts.ifile is not None:
            name, opts.rtype = get_next_domain(input)
525
526

        if connection.dot:
Alexandre's avatar
Alexandre committed
527
            request = remoh.RequestDOT(name, qtype=opts.rtype, use_edns=opts.edns,
528
529
                     want_dnssec=opts.dnssec, no_ecs=opts.no_ecs)
        else:
Alexandre's avatar
Alexandre committed
530
            request = remoh.RequestDOH(name, qtype=opts.rtype, use_edns=opts.edns,
531
532
533
                     want_dnssec=opts.dnssec, no_ecs=opts.no_ecs)

        request.to_wire()
Alexandre's avatar
Alexandre committed
534
        request.i = i
535

Alexandre's avatar
Alexandre committed
536
537
538
        if not opts.dot:
            request.head = opts.head
            request.post = opts.post
539

Alexandre's avatar
Alexandre committed
540
541
542
543
544
        if opts.pipelining: # We do pipelining (DoT)
            connection.pipelining_add_request(request)
        elif opts.multistreams: # We do multistreams (DoH)
            connection.multistreams_add_request(request)
        else:
Alexandre's avatar
Alexandre committed
545
            try:
Alexandre's avatar
Alexandre committed
546
                connection.do_test(request) # perform the query
Alexandre's avatar
Alexandre committed
547
            except (OpenSSL.SSL.Error, remoh.ConnectionDOTException, remoh.DOHException) as e:
Alexandre's avatar
Alexandre committed
548
549
550
                ok = False
                error(e)
                break
Alexandre's avatar
Alexandre committed
551
552
            ok = request.success
            print_result(connection, request)
Alexandre's avatar
Alexandre committed
553
554
555
556
            if opts.tests > 1 and i == 0:
                start2 = time.time()
            if opts.delay is not None:
                time.sleep(opts.delay)
557

Alexandre's avatar
Alexandre committed
558
    if opts.multistreams:
559
        connection.perform_multi(opts.show_time, display_results=opts.display_results)
Alexandre's avatar
Alexandre committed
560
    elif opts.pipelining:
Alexandre's avatar
Alexandre committed
561
        done = 0
Alexandre's avatar
Alexandre committed
562
563
        try:
            current = connection.pipelining_init_pending(opts.max_in_flight)
Alexandre's avatar
Alexandre committed
564
        except remoh.ConnectionDOTException as e:
Alexandre's avatar
Alexandre committed
565
            ok = False
Alexandre's avatar
Alexandre committed
566
            error("%s, %i/%i requests never got a reply" % (e, opts.tests - connection.nbr_finished_queries, opts.tests))
Alexandre's avatar
Alexandre committed
567
568
        else:
            while done < opts.tests:
Alexandre's avatar
Alexandre committed
569
                if time.time() > start + remoh.MAX_DURATION: # if we send thousands of requests
Alexandre's avatar
Alexandre committed
570
571
572
573
574
                                                       # MAX_DURATION will be reached
                                                       # need to increase MAX_DURATION based
                                                       # on the number of queries
                                                       # or to define a relation such as
                                                       # f(tests) = MAX_DURATION
Alexandre's avatar
Alexandre committed
575
                    error("Elapsed time too long, %i/%i requests never got a reply" % (opts.tests-done, opts.tests))
Alexandre's avatar
Alexandre committed
576
577
578
579
                    ok = False
                    break
                id = connection.read_result(connection, connection.pending, display_results=opts.display_results)
                if id is None: # Probably a timeout
Alexandre's avatar
Alexandre committed
580
                    time.sleep(remoh.SLEEP_TIMEOUT)
Alexandre's avatar
Alexandre committed
581
582
583
584
585
586
587
588
                    continue
                done += 1
                over, rank, request = connection.pending[id]
                if not over:
                    error("Internal error, request %i should be over" % id)
                if current < len(connection.all_requests):
                    try:
                        connection.pipelining_fill_pending(current)
Alexandre's avatar
Alexandre committed
589
                    except remoh.ConnectionDOTException as e:
Alexandre's avatar
Alexandre committed
590
                        ok = False
Alexandre's avatar
Alexandre committed
591
                        error("%s, %i/%i requests never got a reply" % (e, opts.tests - connection.nbr_finished_queries, opts.tests))
Alexandre's avatar
Alexandre committed
592
593
                        break
                    current += 1
594

595
    stop = time.time()
Alexandre's avatar
Alexandre committed
596

Alexandre's avatar
Alexandre committed
597
598
    n_queries = connection.nbr_finished_queries

599
600
    if n_queries > 1 and not opts.pipelining and not opts.multistreams:
        extra = ", %.2f ms/request if we ignore the first one" % ((stop-start2)*1000/(n_queries-1))
601
602
    else:
        extra = ""
603
    if not opts.check or opts.verbose:
604
        time_tot = stop - start
Alexandre's avatar
Alexandre committed
605
606
607
608
609
        if n_queries > 1:
            time_per_request = " (%.2f ms/request%s)" % (time_tot / n_queries * 1000, extra)
        else:
            time_per_request = ""
        print("\nTotal elapsed time: %.2f seconds%s" % (time_tot, time_per_request))
Alexandre's avatar
Alexandre committed
610

611
612
    if opts.multistreams and opts.verbose:
        for rcode, n in conn.finished['http'].items():
613
            print("HTTP %d : %d %.2f%%" % (rcode, n, n / n_queries * 100))
Alexandre's avatar
Alexandre committed
614

Alexandre's avatar
Alexandre committed
615
616
    return ok

617
# Main program
618
url, name = parse_opts(opts)
619

620
621
# retrieve all ips when using --check
# not necessary if connectTo is already defined
Alexandre's avatar
Alexandre committed
622
623
if not opts.check or opts.connectTo is not None:
    ip_set = {opts.connectTo, }
624
else:
Alexandre's avatar
Alexandre committed
625
    if opts.dot:
Alexandre's avatar
Alexandre committed
626
627
        port = remoh.PORT_DOT
        if not remoh.is_valid_hostname(url):
Alexandre's avatar
Alexandre committed
628
            error_and_exit("DoT requires a host name or IP address, not \"%s\"" % url)
629
630
        netloc = url
    else:
Alexandre's avatar
Alexandre committed
631
632
        port = remoh.PORT_DOH
        if not remoh.is_valid_url(url):
Alexandre's avatar
Alexandre committed
633
            error_and_exit("DoH requires a valid HTTPS URL, not \"%s\"" % url)
Alexandre's avatar
Alexandre committed
634
635
636
637
638
        try:
            url_parts = urllib.parse.urlparse(url) # A very poor validation, many
            # errors (for instance whitespaces, IPv6 address litterals without
            # brackets...) are ignored.
        except ValueError:
Alexandre's avatar
Alexandre committed
639
            error_and_exit("The provided url \"%s\" could not be parsed" % url)
640
        netloc = url_parts.netloc
Alexandre's avatar
Alexandre committed
641
    if opts.forceIPv4:
642
        family = socket.AF_INET
Alexandre's avatar
Alexandre committed
643
    elif opts.forceIPv6:
644
645
646
        family = socket.AF_INET6
    else:
        family = 0
Alexandre's avatar
Alexandre committed
647
    ip_set = resolved_ips(netloc, port, family, opts.dot)
648

Alexandre's avatar
Alexandre committed
649
650
651
652
653
# print number of IPs found
if opts.verbose and opts.check:
    print("Checking \"%s\" ..." % url)
    print("%d IP found : %s" % (len(ip_set), ', '.join(ip_set)))

Stephane Bortzmeyer's avatar
Stephane Bortzmeyer committed
654
ok = True
Alexandre's avatar
Alexandre committed
655
i = 0 # ip counter
Alexandre's avatar
Alexandre committed
656
for ip in ip_set:
Alexandre's avatar
Alexandre committed
657
    i += 1
Alexandre's avatar
Alexandre committed
658
659
    if opts.dot and opts.vhostname is not None:
        extracheck = opts.vhostname
660
661
    else:
        extracheck = None
Alexandre's avatar
Alexandre committed
662
    if opts.verbose and opts.check and ip:
Alexandre's avatar
Alexandre committed
663
        print("(%d/%d) checking IP : %s" % (i, len(ip_set), ip))
664
    try:
Alexandre's avatar
Alexandre committed
665
        if opts.dot:
Alexandre's avatar
Alexandre committed
666
            conn = remoh.ConnectionDOT(url, servername=extracheck, connect_to=ip,
667
668
669
                                 forceIPv4=opts.forceIPv4, forceIPv6=opts.forceIPv6,
                                 insecure=opts.insecure, verbose=opts.verbose, debug=opts.debug,
                                 sni=opts.sni, key=opts.key, pipelining=opts.pipelining)
670
        else:
Alexandre's avatar
Alexandre committed
671
            conn = remoh.ConnectionDOH(url, servername=extracheck, connect_to=ip,
672
673
674
                                 forceIPv4=opts.forceIPv4, forceIPv6=opts.forceIPv6,
                                 insecure=opts.insecure, verbose=opts.verbose, debug=opts.debug,
                                 multistreams=opts.multistreams)
675
676
    except TimeoutError:
        error("timeout")
Alexandre's avatar
Alexandre committed
677
678
        ok = False
        continue
679
680
    except ConnectionRefusedError:
        error("Connection to server refused")
Alexandre's avatar
Alexandre committed
681
682
        ok = False
        continue
683
    except ValueError:
684
        error("\"%s\" not a name or an IP address" % url)
Alexandre's avatar
Alexandre committed
685
686
        ok = False
        continue
687
    except socket.gaierror:
688
        error("Could not resolve \"%s\"" % url)
Alexandre's avatar
Alexandre committed
689
690
        ok = False
        continue
Alexandre's avatar
Alexandre committed
691
    except remoh.ConnectionDOTException as e:
692
693
694
695
        print(e, file=sys.stderr)
        err = "Could not connect to \"%s\"" % url
        if opts.connectTo is not None:
            err += " on %s" % opts.connectTo
Alexandre's avatar
Alexandre committed
696
697
        elif ip is not None:
            err += " on %s" % ip
698
        error(err)
Alexandre's avatar
Alexandre committed
699
700
        ok = False
        continue
Alexandre's avatar
Alexandre committed
701
    except (remoh.ConnectionException, remoh.DOHException) as e:
702
        error(e)
Alexandre's avatar
Alexandre committed
703
704
705
        ok = False
        continue

Alexandre's avatar
Alexandre committed
706
    if conn.dot and not conn.success:
707
        ok = False
Alexandre's avatar
Alexandre committed
708
        continue
Alexandre's avatar
Alexandre committed
709

Alexandre's avatar
Alexandre committed
710
711
    if opts.ifile is not None:
        input = open(opts.ifile)
Alexandre's avatar
Alexandre committed
712

Alexandre's avatar
Alexandre committed
713
    if not opts.check:
Alexandre's avatar
Alexandre committed
714
        ok = run_default(name, conn, opts)
715
    else:
716
        ok = run_check(conn) and ok # need to run run_check first
Alexandre's avatar
Alexandre committed
717

Alexandre's avatar
Alexandre committed
718
    if opts.ifile is not None:
719
        input.close()
720

Alexandre's avatar
Alexandre committed
721
722
    if conn.state == 'CONN_OK':
        conn.end()
Alexandre's avatar
Alexandre committed
723

Stephane Bortzmeyer's avatar
Stephane Bortzmeyer committed
724
if ok:
Alexandre's avatar
Alexandre committed
725
    if opts.check or opts.pipelining:
Alexandre's avatar
Alexandre committed
726
        print('OK')
727
    sys.exit(0)
Stephane Bortzmeyer's avatar
Stephane Bortzmeyer committed
728
else:
Alexandre's avatar
Alexandre committed
729
730
    if opts.check:
        print('KO')
731
    sys.exit(1)