New mandatory level : nocrash

This is to use with tests that are not documented in the RFC and where
the response of the server is unknown. We think it is good that the server
do not crash when facing those tests.

README.md

@@ -100,7 +100,7 @@ N lines of the FILE
 (read the first line only, use --repeat N to read up to N lines of the file)
 * `--check` : perform a set of predefined tests
 * `--mandatory-level LEVEL` : define the LEVEL of test to perform (only
-with --check). Available LEVEL : `legal`, `necessary`, `nicetohave`
+with --check). Available LEVEL : `legal`, `necessary`, `nicetohave`, `nocrash`
 * `--no-display-results` : disable output of DNS response, this can be combined
 with `-v` to keep only part of the output
 * `-V --vhost <vhost>` : define a specific virtual host
@@ -200,7 +200,7 @@ the choosen level is lower than the level of the test.
 | level | test |
 | ----- | ---- |
 | legal | two queries on the same connection |
-| nocrash¹ | truncated query |
+| nocrash | truncated query |
 
 #### List of tests for DoH
 
@@ -209,13 +209,9 @@ the choosen level is lower than the level of the test.
 | legal | HTTP GET method |
 | legal | HTTP POST method |
 | nicetohave | HTTP HEAD method |
-| nocrash¹ | truncated query |
-| nocrash¹ | Accept-header: text/html |
-| nocrash¹ | Content-type: text/html |
-
-¹ The `nocrash` level is not defined as such in Homer, it is just to
-show that other tests are performed to assess the robustess of the
-DoT/DoH server.
+| nocrash | truncated query |
+| nocrash | Accept-header: text/html |
+| nocrash | Content-type: text/html |
 
 ### Multistreams
 

homer.py

@@ -342,21 +342,19 @@ def run_check(connection):
         ok = check_dot_two_requests(connection, opts)
     else:
         ok = check_doh_methods(connection, opts)
-    if not ok:
+    if not ok and opts.mandatory_level >= homer.mandatory_levels["nicetohave"]:
         return False
 
     # Test that different Header values are not breaking anything
-    # this can be added in a specific level 'donotbreak'
     if not connection.dot:
         # The DoH server is right to reject these (Example: 'HTTP
         # error 415: only Content-Type: application/dns-message is
         # supported')
-        ok = check_doh_header(connection, opts, level=10, accept="text/html") and ok
-        ok = check_doh_header(connection, opts, level=10, content_type="text/html") and ok
+        ok = check_doh_header(connection, opts, level=homer.mandatory_levels["nocrash"], accept="text/html") and ok
+        ok = check_doh_header(connection, opts, level=homer.mandatory_levels["nocrash"], content_type="text/html") and ok
 
     # test if a truncated query breaks anything
-    # again iwbn to have a level such as 'donotbreak' for it
-    ok = check_truncated_query(connection, opts, level=30) and ok
+    ok = check_truncated_query(connection, opts, level=homer.mandatory_levels["nocrash"]) and ok
 
     return ok
 

homer/__init__.py

@@ -40,5 +40,9 @@ DOH_GET = 0
 DOH_POST = 1
 DOH_HEAD = 2
 # Is the test mandatory?
-mandatory_levels = {"legal": 30, "necessary": 20, "nicetohave": 10}
+# legal : RFC compliant
+# necessary : should work
+# nicetohave : not mentionned in the RFC but good if implemented
+# nocrash : edge tests (undocumented) just to see if the server crash (this would be bad)
+mandatory_levels = {"legal": 30, "necessary": 20, "nicetohave": 10, "nocrash": 5}
 

tests.yaml

@@ -183,6 +183,8 @@ tests:
       timeout: 12
       args:
           - '--check'
+          - '--mandatory-level'
+          - 'nocrash'
           - 'https://doh.42l.fr/dns-query'
           - 'framagit.org'
       partstderr: 'Test truncated data: HTTP error 502'