Commit ca58eb5c authored by Gaël Berthaud-Müller's avatar Gaël Berthaud-Müller Committed by Alexandre
Browse files

update remaining db engines

TODO: attention get_test_history devrait casser
parent 70b7d68d
......@@ -19,7 +19,6 @@ requires qw(
create_db
from_config
get_test_history
process_unfinished_tests_give_up
recent_test_hash_id
select_unfinished_tests
test_progress
......@@ -312,33 +311,27 @@ sub process_unfinished_tests {
);
while ( my $h = $sth1->fetchrow_hashref ) {
$self->force_end_test($h->{hash_id}, $h->{results}, $test_run_timeout);
$self->force_end_test($h->{hash_id}, $test_run_timeout);
}
}
sub force_end_test {
my ( $self, $hash_id, $results, $timestamp ) = @_;
my $result;
if ( defined $results && $results =~ /^\[/ ) {
$result = decode_json( $results );
}
else {
$result = [];
}
push @$result,
{
"level" => "CRITICAL",
"module" => "BACKEND_TEST_AGENT",
"tag" => "UNABLE_TO_FINISH_TEST",
"timestamp" => $timestamp,
};
$self->process_unfinished_tests_give_up($result, $hash_id);
my ( $self, $hash_id, $timestamp ) = @_;
$self->add_result_entry( $hash_id, {
timestamp => $timestamp,
module => 'BACKEND_TEST_AGENT',
testcase => '',
tag => 'UNABLE_TO_FINISH_TEST',
level => 'CRITICAL',
args => {},
});
$self->test_progress($hash_id, 100);
}
sub process_dead_test {
my ( $self, $hash_id ) = @_;
my ( $results ) = $self->dbh->selectrow_array("SELECT results FROM test_results WHERE hash_id = ?", undef, $hash_id);
$self->force_end_test($hash_id, $results, $self->get_relative_start_time($hash_id));
$self->force_end_test($hash_id, $self->get_relative_start_time($hash_id));
}
# A thin wrapper around DBI->connect to ensure similar behavior across database
......@@ -475,6 +468,24 @@ sub undelegated {
return 0;
}
sub add_result_entry {
my ( $self, $hash_id, $entry ) = @_;
my $json = JSON::PP->new->allow_blessed->convert_blessed->canonical;
my $nb_inserted = $self->dbh->do(
"INSERT INTO result_entries (hash_id, level, module, testcase, tag, timestamp, args) VALUES (?, ?, ?, ?, ?, ?, ?)",
undef,
$hash_id,
$entry->{level},
$entry->{module},
$entry->{testcase},
$entry->{tag},
$entry->{timestamp},
$json->encode( $entry->{args} ),
);
return $nb_inserted;
}
no Moose::Role;
......
......@@ -116,6 +116,28 @@ sub create_db {
);
}
$dbh->do(
'CREATE TABLE IF NOT EXISTS result_entries (
id integer AUTO_INCREMENT PRIMARY KEY,
hash_id VARCHAR(16) not null,
level varchar(15) not null,
module varchar(255) not null,
testcase varchar(255) not null,
tag varchar(255) not null,
timestamp real not null,
args blob not null
) ENGINE=InnoDB
'
);
$dbh->do(
'CREATE INDEX result_entries__hash_id ON result_entries (hash_id)'
);
$dbh->do(
'CREATE INDEX result_entries__level ON result_entries (level)'
);
####################################################################
# BATCH JOBS
......@@ -163,6 +185,9 @@ sub test_progress {
if ($progress == 1) {
$dbh->do( "UPDATE test_results SET progress=?, test_start_time=NOW() WHERE hash_id=? AND progress <> 100", undef, $progress, $test_id );
}
elsif ($progress == 100) {
$dbh->do( "UPDATE test_results SET progress=?, test_end_time=NOW() WHERE hash_id=? AND progress <> 100", undef, $progress, $test_id );
}
else {
$dbh->do( "UPDATE test_results SET progress=? WHERE hash_id=? AND progress <> 100", undef, $progress, $test_id );
}
......@@ -174,28 +199,28 @@ sub test_progress {
}
sub test_results {
my ( $self, $test_id, $new_results ) = @_;
if ( $new_results ) {
$self->dbh->do( qq[UPDATE test_results SET progress=100, test_end_time=NOW(), results = ? WHERE hash_id=? AND progress < 100],
undef, $new_results, $test_id );
}
my ( $self, $test_id ) = @_;
my $result;
my ( $hrefs ) = $self->dbh->selectall_hashref( "SELECT id, hash_id, CONVERT_TZ(`creation_time`, \@\@session.time_zone, '+00:00') AS creation_time, params, results FROM test_results WHERE hash_id=?", 'hash_id', undef, $test_id );
my ( $hrefs ) = $self->dbh->selectall_hashref( "SELECT id, hash_id, CONVERT_TZ(`creation_time`, \@\@session.time_zone, '+00:00') AS creation_time, params FROM test_results WHERE hash_id=?", 'hash_id', undef, $test_id );
$result = $hrefs->{$test_id};
die Zonemaster::Backend::Error::ResourceNotFound->new( message => "Test not found", data => { test_id => $test_id } )
unless defined $result;
my @result_entries = $self->dbh->selectall_array( "SELECT level, module, testcase, tag, timestamp, args FROM result_entries WHERE hash_id=?", { Slice => {} }, $test_id );
eval {
$result->{params} = decode_json( $result->{params} );
if (defined $result->{results}) {
$result->{results} = decode_json( $result->{results} );
} else {
$result->{results} = [];
}
@result_entries = map {
{
%$_,
args => decode_json( $_->{args} ),
}
} @result_entries;
$result->{results} = \@result_entries;
};
die Zonemaster::Backend::Error::JsonError->new( reason => "$@", data => { test_id => $test_id } )
......@@ -204,76 +229,54 @@ sub test_results {
return $result;
}
sub get_test_history {
my ( $self, $p ) = @_;
my @results;
my $sth = {};
my $dbh = $self->dbh;
my $undelegated = "";
if ($p->{filter} eq "undelegated") {
$undelegated = 1;
$undelegated = "AND undelegated = 1";
} elsif ($p->{filter} eq "delegated") {
$undelegated = 0;
}
if ($p->{filter} eq "all") {
$sth = $self->dbh->prepare(
q[SELECT
id,
hash_id,
CONVERT_TZ(`creation_time`, @@session.time_zone, '+00:00') AS creation_time,
params,
undelegated,
results
FROM
test_results
WHERE
domain = ?
ORDER BY id DESC
LIMIT ? OFFSET ?]
);
$sth->execute( $p->{frontend_params}{domain}, $p->{limit}, $p->{offset} );
} else {
$sth = $self->dbh->prepare(
q[SELECT
id,
hash_id,
CONVERT_TZ(`creation_time`, @@session.time_zone, '+00:00') AS creation_time,
params,
undelegated,
results
FROM
test_results
WHERE
domain = ?
AND undelegated = ?
ORDER BY id DESC
LIMIT ? OFFSET ?]
);
$sth->execute( $p->{frontend_params}{domain}, $undelegated, $p->{limit}, $p->{offset} );
$undelegated = "AND undelegated = 0";
}
while ( my $h = $sth->fetchrow_hashref ) {
$h->{results} = decode_json($h->{results}) if $h->{results};
$h->{params} = decode_json($h->{params}) if $h->{params};
my $critical = ( grep { $_->{level} eq 'CRITICAL' } @{ $h->{results} } );
my $error = ( grep { $_->{level} eq 'ERROR' } @{ $h->{results} } );
my $warning = ( grep { $_->{level} eq 'WARNING' } @{ $h->{results} } );
# More important overwrites
my $overall = 'ok';
$overall = 'warning' if $warning;
$overall = 'error' if $error;
$overall = 'critical' if $critical;
my @results;
my $query = "
SELECT
(SELECT count(*) FROM result_entries where result_entries.hash_id = test_results.hash_id AND level = 'CRITICAL') AS nb_critical,
(SELECT count(*) FROM result_entries where result_entries.hash_id = test_results.hash_id AND level = 'ERROR') AS nb_error,
(SELECT count(*) FROM result_entries where result_entries.hash_id = test_results.hash_id AND level = 'WARNING') AS nb_warning,
id,
hash_id,
CONVERT_TZ(`creation_time`, \@\@session.time_zone, '+00:00') AS creation_time
FROM test_results
WHERE domain = " . $dbh->quote( $p->{frontend_params}->{domain} ) . " $undelegated
ORDER BY id DESC
LIMIT $p->{limit} OFFSET $p->{offset}";
my $sth1 = $dbh->prepare( $query );
$sth1->execute;
while ( my $h = $sth1->fetchrow_hashref ) {
my $overall_result = 'ok';
if ( $h->{nb_critical} ) {
$overall_result = 'critical';
}
elsif ( $h->{nb_error} ) {
$overall_result = 'error';
}
elsif ( $h->{nb_warning} ) {
$overall_result = 'warning';
}
push(
@results,
{
id => $h->{hash_id},
creation_time => $h->{creation_time},
#TODO: undelegated pas dans les resultats de la requete
undelegated => $h->{undelegated},
overall_result => $overall,
overall_result => $overall_result,
}
);
}
......@@ -359,12 +362,6 @@ sub select_unfinished_tests {
}
}
sub process_unfinished_tests_give_up {
my ( $self, $result, $hash_id ) = @_;
$self->dbh->do("UPDATE test_results SET progress = 100, test_end_time = NOW(), results = ? WHERE hash_id=?", undef, encode_json($result), $hash_id);
}
sub get_relative_start_time {
my ( $self, $hash_id ) = @_;
......
......@@ -101,6 +101,27 @@ sub create_db {
'CREATE INDEX IF NOT EXISTS test_results__progress_priority_id ON test_results (progress, priority DESC, id) WHERE (progress = 0)'
);
$dbh->do(
'CREATE TABLE IF NOT EXISTS result_entries (
id serial primary key,
hash_id VARCHAR(16) not null,
level varchar(15) not null,
module varchar(255) not null,
testcase varchar(255) not null,
tag varchar(255) not null,
timestamp real not null,
args json not null
)
'
);
$dbh->do(
'CREATE INDEX result_entries__hash_id ON result_entries (hash_id)'
);
$dbh->do(
'CREATE INDEX result_entries__level ON result_entries (level)'
);
####################################################################
# BATCH JOBS
......@@ -148,8 +169,9 @@ sub test_progress {
if ( $progress ) {
if ($progress == 1) {
$dbh->do( "UPDATE test_results SET progress=?, test_start_time=NOW() WHERE hash_id=? AND progress <> 100", undef, $progress, $test_id );
}
else {
} elsif ($progress == 100) {
$dbh->do( "UPDATE test_results SET progress=?, test_end_time=NOW() WHERE hash_id=? AND progress <> 100", undef, $progress, $test_id );
} else {
$dbh->do( "UPDATE test_results SET progress=? WHERE hash_id=? AND progress <> 100", undef, $progress, $test_id );
}
}
......@@ -160,13 +182,9 @@ sub test_progress {
}
sub test_results {
my ( $self, $test_id, $results ) = @_;
my ( $self, $test_id ) = @_;
my $dbh = $self->dbh;
$dbh->do( "UPDATE test_results SET progress=100, test_end_time=NOW(), results = ? WHERE hash_id=? AND progress < 100",
undef, $results, $test_id )
if ( $results );
my $result;
my ( $hrefs ) = $dbh->selectall_hashref( "SELECT id, hash_id, creation_time at time zone current_setting('TIMEZONE') at time zone 'UTC' as creation_time, params FROM test_results WHERE hash_id=?", 'hash_id', undef, $test_id );
......@@ -218,9 +236,9 @@ sub get_test_history {
my @results;
my $query = "
SELECT
(SELECT count(*) FROM result_entries where result_entries.hash_id = test_results.hash_id AND level = 5) AS nb_critical,
(SELECT count(*) FROM result_entries where result_entries.hash_id = test_results.hash_id AND level = 4) AS nb_error,
(SELECT count(*) FROM result_entries where result_entries.hash_id = test_results.hash_id AND level = 3) AS nb_warning,
(SELECT count(*) FROM result_entries where result_entries.hash_id = test_results.hash_id AND level = 'CRITICAL') AS nb_critical,
(SELECT count(*) FROM result_entries where result_entries.hash_id = test_results.hash_id AND level = 'ERROR') AS nb_error,
(SELECT count(*) FROM result_entries where result_entries.hash_id = test_results.hash_id AND level = 'WARNING') AS nb_warning,
id,
hash_id,
undelegated,
......@@ -248,6 +266,7 @@ sub get_test_history {
{
id => $h->{hash_id},
creation_time => $h->{creation_time},
#TODO: undelegated pas dans les resultats de la requete
undelegated => $h->{undelegated},
overall_result => $overall_result,
}
......@@ -338,34 +357,12 @@ sub select_unfinished_tests {
}
}
sub process_unfinished_tests_give_up {
my ( $self, $result, $hash_id ) = @_;
$self->dbh->do("UPDATE test_results SET progress = 100, test_end_time = NOW(), results = ? WHERE hash_id=?", undef, encode_json($result), $hash_id);
}
sub get_relative_start_time {
my ( $self, $hash_id ) = @_;
return $self->dbh->selectrow_array("SELECT EXTRACT(EPOCH FROM now() - test_start_time) FROM test_results WHERE hash_id=?", undef, $hash_id);
}
sub add_result_entry {
my ( $self, $hash_id, $entry ) = @_;
my $nb_inserted = $self->dbh->do(
"INSERT INTO result_entries (hash_id, level, module, testcase, tag, timestamp, args) VALUES (?, ?, ?, ?, ?, ?, ?)",
undef,
$hash_id,
$entry->{level},
$entry->{module},
$entry->{testcase},
$entry->{tag},
$entry->{timestamp},
encode_json( $entry->{args} ),
);
return $nb_inserted;
}
no Moose;
__PACKAGE__->meta()->make_immutable();
......
......@@ -91,6 +91,27 @@ sub create_db {
'CREATE INDEX IF NOT EXISTS test_results__domain_undelegated ON test_results (domain, undelegated)'
);
$dbh->do(
'CREATE TABLE IF NOT EXISTS result_entries (
id integer PRIMARY KEY AUTOINCREMENT,
hash_id VARCHAR(16) not null,
level varchar(15) not null,
module varchar(255) not null,
testcase varchar(255) not null,
tag varchar(255) not null,
timestamp real not null,
args blob not null
)
'
);
$dbh->do(
'CREATE INDEX result_entries__hash_id ON result_entries (hash_id)'
);
$dbh->do(
'CREATE INDEX result_entries__level ON result_entries (level)'
);
####################################################################
# BATCH JOBS
......@@ -143,6 +164,9 @@ sub test_progress {
if ($progress == 1) {
$dbh->do( "UPDATE test_results SET progress=?, test_start_time=datetime('now') WHERE hash_id=? AND progress <> 100", undef, $progress, $test_id );
}
elsif ($progress == 100) {
$dbh->do( "UPDATE test_results SET progress=?, test_end_time=datetime('now') WHERE hash_id=? AND progress <> 100", undef, $progress, $test_id );
}
else {
$dbh->do( "UPDATE test_results SET progress=? WHERE hash_id=? AND progress <> 100", undef, $progress, $test_id );
}
......@@ -154,28 +178,28 @@ sub test_progress {
}
sub test_results {
my ( $self, $test_id, $new_results ) = @_;
if ( $new_results ) {
$self->dbh->do( qq[UPDATE test_results SET progress=100, test_end_time=datetime('now'), results = ? WHERE hash_id=? AND progress < 100],
undef, $new_results, $test_id );
}
my ( $self, $test_id ) = @_;
my $result;
my ( $hrefs ) = $self->dbh->selectall_hashref( "SELECT id, hash_id, creation_time, params, results FROM test_results WHERE hash_id=?", 'hash_id', undef, $test_id );
my ( $hrefs ) = $self->dbh->selectall_hashref( "SELECT id, hash_id, creation_time, params FROM test_results WHERE hash_id=?", 'hash_id', undef, $test_id );
$result = $hrefs->{$test_id};
die Zonemaster::Backend::Error::ResourceNotFound->new( message => "Test not found", data => { test_id => $test_id } )
unless defined $result;
my @result_entries = $self->dbh->selectall_array( "SELECT level, module, testcase, tag, timestamp, args FROM result_entries WHERE hash_id=?", { Slice => {} }, $test_id );
eval {
$result->{params} = decode_json( $result->{params} );
if (defined $result->{results}) {
$result->{results} = decode_json( $result->{results} );
} else {
$result->{results} = [];
}
@result_entries = map {
{
%$_,
args => decode_json( $_->{args} ) ,
}
} @result_entries;
$result->{results} = \@result_entries;
};
die Zonemaster::Backend::Error::JsonError->new( reason => "$@", data => { test_id => $test_id } )
......@@ -187,7 +211,7 @@ sub test_results {
sub get_test_history {
my ( $self, $p ) = @_;
my @results;
my $dbh = $self->dbh;
my $undelegated = "";
if ($p->{filter} eq "undelegated") {
......@@ -196,44 +220,44 @@ sub get_test_history {
$undelegated = "AND undelegated = 0";
}
my $quoted_domain = $self->dbh->quote( $p->{frontend_params}->{domain} );
$quoted_domain =~ s/'/"/g;
my $query = "SELECT
id,
hash_id,
creation_time,
params,
undelegated,
results
FROM
test_results
WHERE
params like '\%\"domain\":$quoted_domain\%'
$undelegated
ORDER BY id DESC LIMIT $p->{limit} OFFSET $p->{offset} ";
my $sth1 = $self->dbh->prepare( $query );
my @results;
my $query = "
SELECT
(SELECT count(*) FROM result_entries where result_entries.hash_id = test_results.hash_id AND level = 'CRITICAL') AS nb_critical,
(SELECT count(*) FROM result_entries where result_entries.hash_id = test_results.hash_id AND level = 'ERROR') AS nb_error,
(SELECT count(*) FROM result_entries where result_entries.hash_id = test_results.hash_id AND level = 'WARNING') AS nb_warning,
id,
hash_id,
creation_time
FROM test_results
WHERE domain = " . $dbh->quote( $p->{frontend_params}->{domain} ) . " $undelegated
ORDER BY id DESC
LIMIT $p->{limit} OFFSET $p->{offset}";
my $sth1 = $dbh->prepare( $query );
$sth1->execute;
while ( my $h = $sth1->fetchrow_hashref ) {
$h->{results} = decode_json($h->{results}) if $h->{results};
my $critical = ( grep { $_->{level} eq 'CRITICAL' } @{ $h->{results} } );
my $error = ( grep { $_->{level} eq 'ERROR' } @{ $h->{results} } );
my $warning = ( grep { $_->{level} eq 'WARNING' } @{ $h->{results} } );
# More important overwrites
my $overall = 'ok';
$overall = 'warning' if $warning;
$overall = 'error' if $error;
$overall = 'critical' if $critical;
push( @results,
{
id => $h->{hash_id},
creation_time => $h->{creation_time},
undelegated => $h->{undelegated},
overall_result => $overall,
}
);
my $overall_result = 'ok';
if ( $h->{nb_critical} ) {
$overall_result = 'critical';
}
elsif ( $h->{nb_error} ) {
$overall_result = 'error';
}
elsif ( $h->{nb_warning} ) {
$overall_result = 'warning';
}
push(
@results,
{
id => $h->{hash_id},
creation_time => $h->{creation_time},
#TODO: undelegated n'est pas dans les resultats de la requete
undelegated => $h->{undelegated},
overall_result => $overall_result,
}
);
}
$sth1->finish;
return \@results;
}
......@@ -316,12 +340,6 @@ sub select_unfinished_tests {
}
}
sub process_unfinished_tests_give_up {
my ( $self, $result, $hash_id ) = @_;
$self->dbh->do("UPDATE test_results SET progress = 100, test_end_time = DATETIME('now'), results = ? WHERE hash_id=?", undef, encode_json($result), $hash_id);
}
sub get_relative_start_time {
my ( $self, $hash_id ) = @_;
......
......@@ -94,21 +94,19 @@ sub run {
sub {
my ( $entry ) = @_;
# TODO: Make minimum level configurable
if ( $entry->numeric_level >= $numeric{INFO} ) {
$log->info($entry->string);
eval {
$self->{_db}->add_result_entry( $test_id, {
timestamp => $entry->timestamp,
module => $entry->module,
testcase => $entry->testcase,
tag => $entry->tag,
level => $entry->numeric_level,
args => $entry->args // {},
});
};
if ($@) {
$log->error($@);
}
$log->debug("Adding result entry in database: " . $entry->string);
$self->{_db}->add_result_entry( $test_id, {
timestamp => $entry->timestamp,
module => $entry->module,
testcase => $entry->testcase,
tag => $entry->tag,
level => $entry->level,
args => $entry->args // {},
});
}
foreach my $trace ( reverse @{ $entry->trace } ) {
......@@ -179,6 +177,7 @@ sub run {
}
# Actually run tests!
die Zonemaster::Backend::Error::Internal->new( reason => "called with results");
eval { Zonemaster::