From f4900ea721dda9067fd00e29d5c78d1c7a1d4809 Mon Sep 17 00:00:00 2001
From: Gerhard Gonter <ggonter@gmail.com>
Date: Sat, 11 Sep 2021 20:01:39 +0200
Subject: [PATCH] live update for block statistics; added "missing" line to
metablock statistics
---
eprints1.pl | 59 +++++++++++++++++++++++++++++++++++++++++++++++------
1 file changed, 53 insertions(+), 6 deletions(-)
diff --git a/eprints1.pl b/eprints1.pl
index 2e32e76..004b162 100755
--- a/eprints1.pl
+++ b/eprints1.pl
@@ -2668,7 +2668,7 @@ sub oma
{
$db_ot2ut= IRMA::db::get_any_db($cnf, 'ot2ut_database') unless (defined ($db_ot2ut));
- my $col_sync= $db_ot2ut->get_collection('sync');
+# my $col_sync= $db_ot2ut->get_collection('sync');
my $col_req= $db_ot2ut->get_collection('requests');
activity({ activity => 'listening' });
@@ -2824,9 +2824,10 @@ sub ot2ut
}
$db_ot2ut= IRMA::db::get_any_db($cnf, 'ot2ut_database') unless (defined ($db_ot2ut));
- my $col_sync= $db_ot2ut->get_collection('sync');
- my $col_att= $db_ot2ut->get_collection('attachments');
+ my $col_sync= $db_ot2ut->get_collection('sync');
+ my $col_att= $db_ot2ut->get_collection('attachments');
my $col_policy_utheses= $db_ot2ut->get_collection('policy.utheses');
+ my $col_blk= $db_ot2ut->get_collection('blocks');
my $redis_connection= get_redis_db($upload_cnf);;
@@ -2899,6 +2900,25 @@ sub ot2ut
print __LINE__, " earlier sync attempt had errors, retrying...\n";
print __LINE__, " sync_info: ", Dumper($sync_info);
$col_sync->remove( { _id => $sync_info->{_id} } );
+
+ if ($sync_info->{error_code} eq 'ok')
+ {
+ my $context;
+ if ($ot2ut_context eq 'ot2ut-test') { $context= 'test'; }
+ elsif ($ot2ut_context eq 'ot2ut-entw') { $context= 'entw'; }
+ elsif ($ot2ut_context eq 'ot2ut-prod') { $context= 'prod'; }
+
+ my $block_nr= int($eprint_id/100);
+ $col_blk->update({ nr => $block_nr, level => 0, context => $context },
+ { nr => $block_nr, level => 0, context => $context, '$inc' => { count_ok => -1 } },
+ { upsert => 0 });
+
+ $block_nr= int($block_nr/100);
+ $col_blk->update({ nr => $block_nr, level => 1, context => $context },
+ { nr => $block_nr, level => 1, context => $context, '$inc' => { count_ok => -1 } },
+ { upsert => 0 });
+ }
+
$sync_info= undef;
}
}
@@ -2968,6 +2988,7 @@ sub ot2ut
};
push (@synced, $el);
$col_sync->insert($el);
+ # NOTE: do not increment block counters, as this is an error
send_message("upload error: eprint_id=[$eprint_id] eprint_status=[$eprint_status] lastmod=[$lastmod] [conversion errors]");
my $utheses_errors_json_path= 'othes/utheses_json/errors/' . $eprint_id . '.json';
@@ -3029,6 +3050,7 @@ sub ot2ut
$col_sync->insert($el);
$cnt_errors_upload++;
# sleep(2);
+ # NOTE: do not increment block counters, as this is an error
}
else
{
@@ -3237,6 +3259,21 @@ sub ot2ut
$col_sync->insert($out_row);
send_message("upload $upload_success: eprint_id=[$eprint_id] eprint_status=[$eprint_status] lastmod=[$lastmod] context=[$ot2ut_context] utheses_id=[$utheses_id] time_total=$td_start time_upload=$td_curl") unless ($silent_upload_success && $upload_success eq 'ok');
+
+ my $context;
+ if ($ot2ut_context eq 'ot2ut-test') { $context= 'test'; }
+ elsif ($ot2ut_context eq 'ot2ut-entw') { $context= 'entw'; }
+ elsif ($ot2ut_context eq 'ot2ut-prod') { $context= 'prod'; }
+ my $block_nr= int($eprint_id/100);
+ $col_blk->update({ nr => $block_nr, level => 0, context => $context },
+ { '$inc' => { count_ok => 1 } },
+ { upsert => 1 });
+
+ $block_nr= int($block_nr/100);
+ $col_blk->update({ nr => $block_nr, level => 1, context => $context },
+ { '$inc' => { count_ok => 1 } },
+ { upsert => 1 });
+
}
else
{
@@ -4622,7 +4659,7 @@ sub update_policies
$db_ot2ut= IRMA::db::get_any_db($cnf, 'ot2ut_database') unless (defined ($db_ot2ut));
my $col_utp= $db_ot2ut->get_collection('utheses.policies');
- if (defined ($refresh_oldest_policies) && $refresh_oldest_policies > 0 && $refresh_oldest_policies <= 5000)
+ if (defined ($refresh_oldest_policies) && $refresh_oldest_policies > 0 && $refresh_oldest_policies <= 10000)
{
my $c1= $col_utp->find( {}, { '_id' => 1, eprint_id => 1, generated => 1 });
$c1->sort( { generated => 1 } );
@@ -5154,7 +5191,7 @@ EOX
my $mb_html= sprintf("metablock_%d.html", $metablock_nr);
my $mb_html_fp= join('/', $base_path, $mb_html);
- print IDX "<tr>\n <td><a href=\"$mb_html\" target=\"metablocks\">", $metablock_nr, " (${metablock_nr}xxyy)</a></td>\n <td>", $mb_othes, "</td>\n";
+ print IDX "<tr>\n <td><a href=\"$mb_html\" target=\"metablocks\">", $metablock_nr, " (${metablock_nr}yyxx)</a></td>\n <td>", $mb_othes, "</td>\n";
$col_blk->update({ nr => $metablock_nr, level => 1, context => 'othes' },
{ nr => $metablock_nr, level => 1, context => 'othes', count_ok => $mb_othes, count_errors => 0 },
@@ -5218,7 +5255,7 @@ EOX
my $b_html_fp= join('/', $base_path, $b_html);
my $b_othes= $block->{othes}->{cnt_total};
- print MB "<tr>\n <td><a href=\"$b_html\" target=\"blocks\">", $block_nr, " (${block_nr}yy)</a></td>\n <td>", $b_othes, "</td>\n";
+ print MB "<tr>\n <td><a href=\"$b_html\" target=\"blocks\">", $block_nr, " (${block_nr}xx)</a></td>\n <td>", $b_othes, "</td>\n";
$col_blk->update({ nr => $block_nr, level => 0, context => 'othes' },
{ nr => $block_nr, level => 0, context => 'othes', count_ok => $b_othes, count_errors => 0 },
@@ -5364,6 +5401,16 @@ EOX
}
print IDX "</tr>\n";
+ print IDX "<tr>\n <td>missing</td>\n <td> </td>\n";
+ foreach my $context (qw(entw test prod))
+ {
+ my $m= $total_othes - $totals{$context}->{cnt_ok};
+ my $pct= ($total_othes == 0) ? 0 : $m*100.0/$total_othes;
+ my $ck1= ($m == 0) ? 'lightgreen' : 'lightblue';
+ printf IDX (" <td bgcolor=\"$ck1\">%d</td><td bgcolor=\"$ck1\">%7.4f %%</td><td bgcolor=\"$ck1\"> </td>\n", $m, $pct);
+ }
+ print IDX "</tr>\n";
+
print IDX "</table>\n";
# print statistics
--
GitLab