diff --git a/lib/Zonemaster/Backend/DB.pm b/lib/Zonemaster/Backend/DB.pm
index 119ee1823f1a482d1d438f290a2cb8e8372bdcd2..969e28e1be863aa56ead3b7d5b7fc211e125f468 100644
--- a/lib/Zonemaster/Backend/DB.pm
+++ b/lib/Zonemaster/Backend/DB.pm
@@ -857,6 +857,34 @@ sub add_result_entries {
$sth = $sth->execute(map { @$_ } @records);
}
+sub add_result_entries_multiple {
+ my ( $self, %test_results ) = @_;
+ my @records;
+
+ my $json = JSON::PP->new->allow_blessed->convert_blessed->canonical;
+
+ foreach my $hash_id (keys %test_results ) {
+ foreach my $m ( @{$test_results{$hash_id}} ) {
+ my $r = [
+ $hash_id,
+ $m->level,
+ $m->module,
+ $m->testcase,
+ $m->tag,
+ $m->timestamp,
+ $json->encode( $m->args // {} ),
+ ];
+
+ push @records, $r;
+ }
+ }
+
+ my $query_values = join ", ", ("(?, ?, ?, ?, ?, ?, ?)") x @records;
+ my $query = "INSERT INTO result_entries (hash_id, level, module, testcase, tag, timestamp, args) VALUES $query_values";
+ my $sth = $self->dbh->prepare($query);
+ $sth = $sth->execute(map { @$_ } @records);
+}
+
no Moose::Role;
1;
diff --git a/lib/Zonemaster/Backend/RedisQueue.pm b/lib/Zonemaster/Backend/RedisQueue.pm
index 6a1edfc87e51ab93a8e4151b4258e304472b0ccd..7c6b4ea24bd174685cbb2388d442554bf71607b1 100644
--- a/lib/Zonemaster/Backend/RedisQueue.pm
+++ b/lib/Zonemaster/Backend/RedisQueue.pm
@@ -21,6 +21,16 @@ has 'db' => (
required => 1,
);
+=comment
+
+Schema:
+* zm-test-requests -> list of test ids, list of ids of to tests that need to be run
+* zm-test-processing -> list of test ids, list of ids of tests that are currently running
+* zm-test-lock:$test_id -> value, arbitrary value that expired after $test_timeout, used to track test that have timed out.
+* zm-test-params -> hash { test id => json encoded params }
+* zm-test-results -> hash { test id => json encoded result entries}
+=cut
+
sub from_config {
my ( $class, $config ) = @_;
@@ -66,6 +76,38 @@ sub expired_tests {
return @tests;
}
+sub add_result_entries {
+ my ( $self, $hash_id, $entries ) = @_;
+ my $json = JSON::PP->new->allow_blessed->convert_blessed;
+
+ my @out;
+ foreach my $m ( @{$entries} ) {
+ my %r;
+ $r{timestamp} = $m->timestamp;
+ $r{module} = $m->module;
+ $r{testcase} = $m->testcase;
+ $r{tag} = $m->tag;
+ $r{level} = $m->level;
+ $r{args} = $m->args if $m->args;
+
+ push @out, \%r;
+ }
+
+ $self->redis->hset("zm-test-results", $hash_id, $json->encode(\@out));
+}
+
+sub get_test_results {
+ my ( $self ) = @_;
+
+ $self->redis->hgetall( "zm-test-results" );
+}
+
+sub remove_results {
+ my ( $self, $hash_id ) = @_;
+
+ $self->redis->hdel( "zm-test-results", $hash_id );
+}
+
sub queue_batch_job {
my ( $self, $batch_id ) = @_;
my $query = q[
diff --git a/lib/Zonemaster/Backend/TestAgent.pm b/lib/Zonemaster/Backend/TestAgent.pm
index 45fefd1d1595e0a6cf9d6fe7c4cf3ac631b7ff10..1901fd92cb55e13ff910239afc46fe8409089965 100644
--- a/lib/Zonemaster/Backend/TestAgent.pm
+++ b/lib/Zonemaster/Backend/TestAgent.pm
@@ -54,11 +54,11 @@ sub new {
}
sub run {
- my ( $self, $params, $show_progress ) = @_;
+ my ( $self, $queue, $test_id, $show_progress ) = @_;
my @accumulator;
- my $test_id = $params->{hash_id};
+ my $params = $queue->get_test_params( $test_id );
my ( $domain ) = $params->{domain};
if ( !$domain ) {
@@ -158,14 +158,16 @@ sub run {
my @entries = grep { $_->numeric_level >= $numeric{INFO} } @{ Zonemaster::Engine->logger->entries };
Zonemaster::Backend::Metrics::timing("zonemaster.testagent.log_callback_add_result_entry_grep_duration", tv_interval($start_time_2) * 1000);
- $self->{_db}->add_result_entries( $test_id, \@entries);
+ #$self->{_db}->add_result_entries( $test_id, \@entries);
+ $queue->add_result_entries( $test_id, \@entries );
my $callback_add_result_entry_duration = tv_interval($start_time_2);
Zonemaster::Backend::Metrics::timing("zonemaster.testagent.log_callback_add_result_entry_duration", $callback_add_result_entry_duration * 1000);
#$log->debug("Callback timing for $test_id: $callback_duration / $callback_add_result_entry_duration ");
- $self->{_db}->test_progress( $test_id, 100 );
+ #$self->{_db}->test_progress( $test_id, 100 );
+ $queue->test_finished( $test_id );
return;
} ## end sub run
diff --git a/script/zonemaster_backend_results_inserter b/script/zonemaster_backend_results_inserter
new file mode 100644
index 0000000000000000000000000000000000000000..01e04ce4da67b5833ac2e17de2a2d67a1f4a9453
--- /dev/null
+++ b/script/zonemaster_backend_results_inserter
@@ -0,0 +1,71 @@
+#!/usr/bin/env perl
+
+use 5.14.2;
+use warnings;
+
+use Zonemaster::Backend::DB;
+use Zonemaster::Backend::Config;
+use Zonemaster::Backend::Metrics;
+use Zonemaster::Backend::RedisQueue;
+
+use Log::Any qw( $log );
+use Log::Any::Adapter;
+use JSON::PP;
+use Time::HiRes qw[time sleep gettimeofday tv_interval];
+
+
+Log::Any::Adapter->set(
+ '+Zonemaster::Backend::Log',
+ log_level => $ENV{ZM_BACKEND_INSERTER_LOGLEVEL} // "info",
+ json => $ENV{ZM_BACKEND_INSERTER_LOGJSON},
+ stderr => 1
+);
+
+$SIG{__WARN__} = sub {
+ $log->warning(map s/^\s+|\s+$//gr, map s/\n/ /gr, @_);
+};
+
+
+sub main {
+ my $config = Zonemaster::Backend::Config->load_config();
+ my $dbtype = $config->DB_engine;
+ my $dbclass = Zonemaster::Backend::DB->get_db_class( $dbtype );
+ my $db = $dbclass->from_config( $config );
+ Zonemaster::Backend::Metrics->setup($config->METRICS_statsd_host, $config->METRICS_statsd_port);
+
+ $log->info("Starting cleaner process");
+
+ my $queue = Zonemaster::Backend::RedisQueue->from_config( $config );
+ while ( 1 ) {
+ my %test_results = $queue->get_test_results;
+ if ( %test_results ) {
+ my $nb_tests = scalar %test_results;
+ $log->info( "Inserting new results for $nb_tests tests", { test_count => $nb_tests } );
+ my $start_time = [ gettimeofday ];
+
+ %test_results = map {
+ $_ => [
+ map { Zonemaster::Engine::Logger::Entry->new($_) } @{decode_json( $test_results{$_} )}
+ ]
+ } keys %test_results;
+
+ $db->add_result_entries_multiple( %test_results );
+
+ foreach my $hash_id ( keys %test_results ) {
+ $db->test_progress( $hash_id, 100 );
+ Zonemaster::Backend::Metrics::increment("zonemaster.testagent.test_results_inserted");
+ $queue->remove_results( $hash_id );
+ }
+
+ Zonemaster::Backend::Metrics::timing("zonemaster.testagent.test_results_insert_duration_seconds", tv_interval($start_time) * 1000);
+ }
+
+ sleep $config->ZONEMASTER_max_zonemaster_execution_time;
+
+ }
+
+ return;
+}
+
+
+main;
diff --git a/script/zonemaster_backend_testagent b/script/zonemaster_backend_testagent
index 63e98719468f2f12f3e72d8d3744037bc401a7f4..9a382cbf46b440473d38c41a05bcc85204b7be8c 100755
--- a/script/zonemaster_backend_testagent
+++ b/script/zonemaster_backend_testagent
@@ -131,9 +131,7 @@ sub main {
Zonemaster::Backend::Metrics::increment("zonemaster.testagent.tests_started");
my $start_time = [ gettimeofday ];
- my $params = $chlid_queue->get_test_params( $test_id );
- $params->{hash_id} = $test_id;
- eval { $agent->run( $params, $show_progress ) };
+ eval { $agent->run( $chlid_queue, $test_id, $show_progress ) };
if ( $@ ) {
chomp $@;
@@ -144,7 +142,6 @@ sub main {
#$self->db->process_dead_test( $test_id )
}
else {
- $chlid_queue->test_finished( $test_id );
Zonemaster::Backend::Metrics::increment("zonemaster.testagent.tests_completed");
$log->infof( "Test completed: %s", $test_id );
}