Skip to content
Snippets Groups Projects
Commit 3359e4b9 authored by Gerhard Gonter's avatar Gerhard Gonter :speech_balloon:
Browse files

more diagnistic information

parent 989af6d9
Branches
No related tags found
No related merge requests found
......@@ -118,6 +118,7 @@ my $op_mode= 'unknown';
my $ot2ut_context= 'ot2ut-test'; # TODO: parametrize
# my $ot2ut_context= 'ot2ut-prod'; # TODO: parametrize
my $oma_sleep_time= 10;
my $MAX_METABLOCK= 670;
my %map_ot2ut_roles=
(
......@@ -451,7 +452,7 @@ my $db_ot2ut;
my $col_msg;
my $col_activity;
my $last_activity= 0;
my $activity_period= 300;
my $activity_period= 60;
my $cnf= Util::JSON::read_json_file ($config_fnm);
......@@ -2756,7 +2757,7 @@ sub send_message
$col_msg= $db_ot2ut->get_collection('messages') unless (defined ($col_msg));
return undef unless (defined ($col_msg));
print __LINE__, " sending message [$text]\n";
print __LINE__, ' ', scalar localtime(time()), " sending message [$text]\n";
my $msg=
{
message => $text,
......@@ -2845,8 +2846,9 @@ sub ot2ut
}
my @synced= ();
my ($cnt_synced, $cnt_upload_ok, $cnt_errors_data, $cnt_errors_upload, $cnt_errors_ingest, $cnt_skipped)= (0, 0, 0, 0, 0, 0);
my ($cnt_att_synced, $cnt_att_ok, $cnt_att_errors_upload)= (0, 0, 0);
my ($cnt_synced, $cnt_upload_ok, $cnt_errors_data, $cnt_errors_upload, $cnt_errors_ingest, $cnt_errors_attachments, $cnt_skipped)=
(0, 0, 0, 0, 0, 0, 0);
my ($cnt_att_synced, $cnt_att_ok, $cnt_att_errors_upload)= (0, 0, 0); # counters for the whole batch
my $cnt_eprint_ids= @eprint_ids;
print __LINE__, " ot2ut: ot2ut_eprint_status=$ot2ut_eprint_status cnt_eprint_ids=$cnt_eprint_ids MAX_SYNC=$MAX_SYNC\n";
sleep(3);
......@@ -2988,6 +2990,8 @@ sub ot2ut
$result_data= Util::JSON::read_json_file($utheses_upload_result_json_path);
};
my $out_row; # mongodb record in the sync database (or collection)
my ($upload_success, $td_start, $td_curl);
if ($@)
{
print __LINE__, " can't parse upload_result; error=[$@]\n";
......@@ -3015,11 +3019,10 @@ sub ot2ut
my ($status, $import_status, $utheses_id1, $response_msg, $alerts)= map { $result_data->{$_} } qw(status importStatus uthesesId responseMsg alerts);
print __LINE__, " status=[$status] response_msg=[$response_msg]\n";
my $td_start= time()-$t_start;
my $td_curl= time()-$t_curl;
$td_start= time()-$t_start;
$td_curl= time()-$t_curl;
my $upload_success;
my $out_row=
$out_row=
{
eprint_id => $eprint_id,
eprint_status => $eprint_status,
......@@ -3032,7 +3035,7 @@ sub ot2ut
uploaded_fnm => $lfnm,
upload_status => $status,
response_msg => $response_msg,
attachement_cnt => scalar @docs,
attachment_count => scalar @docs,
};
if (defined ($utheses_id1) && $status eq '200')
......@@ -3058,9 +3061,11 @@ sub ot2ut
}
push (@synced, $out_row);
$col_sync->insert($out_row);
send_message("upload $upload_success: eprint_id=[$eprint_id] eprint_status=[$eprint_status] lastmod=[$lastmod] context=[$ot2ut_context] utheses_id=[$utheses_id] time_total=$td_start time_upload=$td_curl") unless ($silent_upload_success && $upload_success eq 'ok');
# moved down
# $col_sync->insert($out_row);
# send_message("upload $upload_success: eprint_id=[$eprint_id] eprint_status=[$eprint_status] lastmod=[$lastmod] context=[$ot2ut_context] utheses_id=[$utheses_id] time_total=$td_start time_upload=$td_curl") unless ($silent_upload_success && $upload_success eq 'ok');
}
# sleep(2);
......@@ -3068,7 +3073,7 @@ sub ot2ut
if (defined ($utheses_id))
{
# process remaining documents as attachments
my $attachment_number= 0;
my ($attachment_number, $attachment_ok, $attachment_error)= (0, 0, 0);
my $attachment_pid;
my $curl_status;
while (my $attachment= shift(@docs))
......@@ -3122,6 +3127,7 @@ sub ot2ut
{
print __LINE__, " can't parse upload_result; error=[$@]\n";
$cnt_att_errors_upload++;
$attachment_error++;
$att_status= $ai->{error_code}= 'upload_error';
$ai->{errors}= [ { error => 'upload_error', error_info => $@ } ];
}
......@@ -3136,11 +3142,13 @@ sub ot2ut
if ($curl_status1 eq '200')
{
$cnt_att_ok++;
$attachment_ok++;
$att_status= $ai->{error_code}= 'ok';
}
else
{
$cnt_att_errors_upload++;
$attachment_error++;
$att_status= $ai->{error_code}= 'ingest_error';
}
......@@ -3194,6 +3202,23 @@ sub ot2ut
$col_att->insert($ai); # NOTE/TODO: no effort is made to check for duplicate uploads of attachments;
} # end of processing for one attachment
# TODO 2020-11-09: check, if uploading attachments returned errors, record this fact in the objects sync record
if ($attachment_number > 0) # any attacments at all
{
# $out_row->{attachment_count}= $attachment_number;
$out_row->{attachment_ok}= $attachment_ok;
$out_row->{attachment_error}= $attachment_error;
if ($attachment_error)
{
$out_row->{error_code}= $upload_success= 'attachment_error';
$cnt_errors_attachments++; # global counter
}
}
$col_sync->insert($out_row);
send_message("upload $upload_success: eprint_id=[$eprint_id] eprint_status=[$eprint_status] lastmod=[$lastmod] context=[$ot2ut_context] utheses_id=[$utheses_id] time_total=$td_start time_upload=$td_curl") unless ($silent_upload_success && $upload_success eq 'ok');
}
else
{
......@@ -3208,7 +3233,7 @@ sub ot2ut
my $res;
if ($cnt_synced)
{
$res= "synced $cnt_synced objects in context $ot2ut_context; data_errors: $cnt_errors_data; upload_errors: $cnt_errors_upload; ingest_errors: $cnt_errors_ingest";
$res= "synced $cnt_synced objects in context $ot2ut_context; data_errors: $cnt_errors_data; upload_errors: $cnt_errors_upload; ingest_errors: $cnt_errors_ingest; attachment_errors: $cnt_errors_attachments";
my $fnm= sprintf('ot2ut_%s.tsv', ts_ISO());
Util::Matrix::save_hash_as_csv(\@ot2ut_synced_columns, \@synced, $fnm, "\t", '', "\n", 1);
print __LINE__, " $res, see [$fnm]\n";
......@@ -3505,6 +3530,17 @@ the structure of json file has slightly changed
# 2020-08-05:
$thesis->{policies}->{fulltext_locked}= 0 if ($eprint_status eq 'archive' && $main_file->{security} eq 'public');
# 2020-11-23: column formatdesc in table document contains notes by UBW staff for that particular document.
# For attachments, these are all but the first document, this is stored as "description" in the attachment's metadata.
# For the main document it self, this information was not stored anywhere.
# Feature Request: store that information into utheses_note_2, e.g. by concatinating it
if (defined ($main_file->{formatdesc}) && $main_file->{formatdesc} ne '')
{
$utp->{utheses_note_2}= (exists($utp->{utheses_note_2}))
? join('; ', $utp->{utheses_note_2}, $main_file->{formatdesc})
: $main_file->{formatdesc};
}
$ut->public('thesis', $thesis);
# Mon May 11 22:12:38 CEST 2020 asked nd about this, especially thesis_doc_added_date:
......@@ -3921,7 +3957,7 @@ mysql> select count(*), abstract_nicht_anzeigen from eprint group by abstract_ni
# diese objekte sollten wir uns notieren. bitte stattdessen einen leeren string uebergeben.
push (@warnings, { warning => 'date_app missing, can not assign assessment_date' });
$thesis{assessment_date}= '';
$thesis{assessment_date}= undef;
}
# print __LINE__, " thesis: ", Dumper (\%thesis);
......@@ -4565,7 +4601,7 @@ sub update_policies
$db_ot2ut= IRMA::db::get_any_db($cnf, 'ot2ut_database') unless (defined ($db_ot2ut));
my $col_utp= $db_ot2ut->get_collection('utheses.policies');
if (defined ($refresh_oldest_policies) && $refresh_oldest_policies > 0 && $refresh_oldest_policies <= 3000)
if (defined ($refresh_oldest_policies) && $refresh_oldest_policies > 0 && $refresh_oldest_policies <= 5000)
{
my $c1= $col_utp->find( {}, { '_id' => 1, eprint_id => 1, generated => 1 });
$c1->sort( { generated => 1 } );
......@@ -4730,17 +4766,18 @@ sub policies_stats
$eprint_ids[$eprint_id]->{$context}= $row_sync;
if (defined ($utheses_id))
if (defined ($utheses_id) && $error_code eq 'ok')
{
$synced{$eprint_id}->{$context}= [ $lastmod, $utheses_id ];
$totals{$context}->{cnt_ok}++;
$blocks[$block_nr]->{$context}->{cnt_ok}++;
$metablocks[$metablock_nr]->{$context}->{cnt_ok}++;
}
else
{
# push (@upload_errors, $row_sync);
push (@{$upload_errors{$error_code}}, $eprint_id);
$totals{$context}->{cnt_error}++;
$blocks[$block_nr]->{$context}->{cnt_error}++;
$metablocks[$metablock_nr]->{$context}->{cnt_error}++;
......@@ -5123,7 +5160,7 @@ EOX
EOX
my $block_start= $metablock_nr*100;
my $block_last= ($metablock_nr == 7) ? 750 : $block_start+99;
my $block_last= ($metablock_nr == 6) ? $MAX_METABLOCK : $block_start+99;
for (my $block_nr= $block_start; $block_nr <= $block_last; $block_nr++)
{
next unless (defined ($blocks[$block_nr]));
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment