Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
I
irma2
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Wiki
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Deploy
Releases
Package registry
Model registry
Operate
Terraform modules
Monitor
Incidents
Service Desk
Analyze
Value stream analytics
Contributor analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
GitLab community forum
Contribute to GitLab
Provide feedback
Terms and privacy
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
Gerhard Gonter
irma2
Commits
3359e4b9
Commit
3359e4b9
authored
Feb 2, 2021
by
Gerhard Gonter
Browse files
Options
Downloads
Patches
Plain Diff
more diagnistic information
parent
989af6d9
Branches
Branches containing commit
No related tags found
No related merge requests found
Changes
1
Show whitespace changes
Inline
Side-by-side
Showing
1 changed file
eprints1.pl
+55
-18
55 additions, 18 deletions
eprints1.pl
with
55 additions
and
18 deletions
eprints1.pl
+
55
−
18
View file @
3359e4b9
...
...
@@ -118,6 +118,7 @@ my $op_mode= 'unknown';
my
$ot2ut_context
=
'
ot2ut-test
';
# TODO: parametrize
# my $ot2ut_context= 'ot2ut-prod'; # TODO: parametrize
my
$oma_sleep_time
=
10
;
my
$MAX_METABLOCK
=
670
;
my
%map_ot2ut_roles
=
(
...
...
@@ -451,7 +452,7 @@ my $db_ot2ut;
my
$col_msg
;
my
$col_activity
;
my
$last_activity
=
0
;
my
$activity_period
=
30
0
;
my
$activity_period
=
6
0
;
my
$cnf
=
Util::JSON::
read_json_file
(
$config_fnm
);
...
...
@@ -2756,7 +2757,7 @@ sub send_message
$col_msg
=
$db_ot2ut
->
get_collection
('
messages
')
unless
(
defined
(
$col_msg
));
return
undef
unless
(
defined
(
$col_msg
));
print
__LINE__
,
"
sending message [
$text
]
\n
";
print
__LINE__
,
'
',
scalar
localtime
(
time
()),
"
sending message [
$text
]
\n
";
my
$msg
=
{
message
=>
$text
,
...
...
@@ -2845,8 +2846,9 @@ sub ot2ut
}
my
@synced
=
();
my
(
$cnt_synced
,
$cnt_upload_ok
,
$cnt_errors_data
,
$cnt_errors_upload
,
$cnt_errors_ingest
,
$cnt_skipped
)
=
(
0
,
0
,
0
,
0
,
0
,
0
);
my
(
$cnt_att_synced
,
$cnt_att_ok
,
$cnt_att_errors_upload
)
=
(
0
,
0
,
0
);
my
(
$cnt_synced
,
$cnt_upload_ok
,
$cnt_errors_data
,
$cnt_errors_upload
,
$cnt_errors_ingest
,
$cnt_errors_attachments
,
$cnt_skipped
)
=
(
0
,
0
,
0
,
0
,
0
,
0
,
0
);
my
(
$cnt_att_synced
,
$cnt_att_ok
,
$cnt_att_errors_upload
)
=
(
0
,
0
,
0
);
# counters for the whole batch
my
$cnt_eprint_ids
=
@eprint_ids
;
print
__LINE__
,
"
ot2ut: ot2ut_eprint_status=
$ot2ut_eprint_status
cnt_eprint_ids=
$cnt_eprint_ids
MAX_SYNC=
$MAX_SYNC
\n
";
sleep
(
3
);
...
...
@@ -2988,6 +2990,8 @@ sub ot2ut
$result_data
=
Util::JSON::
read_json_file
(
$utheses_upload_result_json_path
);
};
my
$out_row
;
# mongodb record in the sync database (or collection)
my
(
$upload_success
,
$td_start
,
$td_curl
);
if
(
$@
)
{
print
__LINE__
,
"
can't parse upload_result; error=[$@]
\n
";
...
...
@@ -3015,11 +3019,10 @@ sub ot2ut
my
(
$status
,
$import_status
,
$utheses_id1
,
$response_msg
,
$alerts
)
=
map
{
$result_data
->
{
$_
}
}
qw(status importStatus uthesesId responseMsg alerts)
;
print
__LINE__
,
"
status=[
$status
] response_msg=[
$response_msg
]
\n
";
my
$td_start
=
time
()
-
$t_start
;
my
$td_curl
=
time
()
-
$t_curl
;
$td_start
=
time
()
-
$t_start
;
$td_curl
=
time
()
-
$t_curl
;
my
$upload_success
;
my
$out_row
=
$out_row
=
{
eprint_id
=>
$eprint_id
,
eprint_status
=>
$eprint_status
,
...
...
@@ -3032,7 +3035,7 @@ sub ot2ut
uploaded_fnm
=>
$lfnm
,
upload_status
=>
$status
,
response_msg
=>
$response_msg
,
attach
e
ment_cnt
=>
scalar
@docs
,
attachment_c
ou
nt
=>
scalar
@docs
,
};
if
(
defined
(
$utheses_id1
)
&&
$status
eq
'
200
')
...
...
@@ -3058,9 +3061,11 @@ sub ot2ut
}
push
(
@synced
,
$out_row
);
$col_sync
->
insert
(
$out_row
);
send_message
("
upload
$upload_success
: eprint_id=[
$eprint_id
] eprint_status=[
$eprint_status
] lastmod=[
$lastmod
] context=[
$ot2ut_context
] utheses_id=[
$utheses_id
] time_total=
$td_start
time_upload=
$td_curl
")
unless
(
$silent_upload_success
&&
$upload_success
eq
'
ok
');
# moved down
# $col_sync->insert($out_row);
# send_message("upload $upload_success: eprint_id=[$eprint_id] eprint_status=[$eprint_status] lastmod=[$lastmod] context=[$ot2ut_context] utheses_id=[$utheses_id] time_total=$td_start time_upload=$td_curl") unless ($silent_upload_success && $upload_success eq 'ok');
}
# sleep(2);
...
...
@@ -3068,7 +3073,7 @@ sub ot2ut
if
(
defined
(
$utheses_id
))
{
# process remaining documents as attachments
my
$attachment_number
=
0
;
my
(
$attachment_number
,
$attachment_ok
,
$attachment_error
)
=
(
0
,
0
,
0
)
;
my
$attachment_pid
;
my
$curl_status
;
while
(
my
$attachment
=
shift
(
@docs
))
...
...
@@ -3122,6 +3127,7 @@ sub ot2ut
{
print
__LINE__
,
"
can't parse upload_result; error=[$@]
\n
";
$cnt_att_errors_upload
++
;
$attachment_error
++
;
$att_status
=
$ai
->
{
error_code
}
=
'
upload_error
';
$ai
->
{
errors
}
=
[
{
error
=>
'
upload_error
',
error_info
=>
$@
}
];
}
...
...
@@ -3136,11 +3142,13 @@ sub ot2ut
if
(
$curl_status1
eq
'
200
')
{
$cnt_att_ok
++
;
$attachment_ok
++
;
$att_status
=
$ai
->
{
error_code
}
=
'
ok
';
}
else
{
$cnt_att_errors_upload
++
;
$attachment_error
++
;
$att_status
=
$ai
->
{
error_code
}
=
'
ingest_error
';
}
...
...
@@ -3194,6 +3202,23 @@ sub ot2ut
$col_att
->
insert
(
$ai
);
# NOTE/TODO: no effort is made to check for duplicate uploads of attachments;
}
# end of processing for one attachment
# TODO 2020-11-09: check, if uploading attachments returned errors, record this fact in the objects sync record
if
(
$attachment_number
>
0
)
# any attacments at all
{
# $out_row->{attachment_count}= $attachment_number;
$out_row
->
{
attachment_ok
}
=
$attachment_ok
;
$out_row
->
{
attachment_error
}
=
$attachment_error
;
if
(
$attachment_error
)
{
$out_row
->
{
error_code
}
=
$upload_success
=
'
attachment_error
';
$cnt_errors_attachments
++
;
# global counter
}
}
$col_sync
->
insert
(
$out_row
);
send_message
("
upload
$upload_success
: eprint_id=[
$eprint_id
] eprint_status=[
$eprint_status
] lastmod=[
$lastmod
] context=[
$ot2ut_context
] utheses_id=[
$utheses_id
] time_total=
$td_start
time_upload=
$td_curl
")
unless
(
$silent_upload_success
&&
$upload_success
eq
'
ok
');
}
else
{
...
...
@@ -3208,7 +3233,7 @@ sub ot2ut
my
$res
;
if
(
$cnt_synced
)
{
$res
=
"
synced
$cnt_synced
objects in context
$ot2ut_context
; data_errors:
$cnt_errors_data
; upload_errors:
$cnt_errors_upload
; ingest_errors:
$cnt_errors_ingest
";
$res
=
"
synced
$cnt_synced
objects in context
$ot2ut_context
; data_errors:
$cnt_errors_data
; upload_errors:
$cnt_errors_upload
; ingest_errors:
$cnt_errors_ingest
; attachment_errors:
$cnt_errors_attachments
";
my
$fnm
=
sprintf
('
ot2ut_%s.tsv
',
ts_ISO
());
Util::Matrix::
save_hash_as_csv
(
\
@ot2ut_synced_columns
,
\
@synced
,
$fnm
,
"
\t
",
'',
"
\n
",
1
);
print
__LINE__
,
"
$res
, see [
$fnm
]
\n
";
...
...
@@ -3505,6 +3530,17 @@ the structure of json file has slightly changed
# 2020-08-05:
$thesis
->
{
policies
}
->
{
fulltext_locked
}
=
0
if
(
$eprint_status
eq
'
archive
'
&&
$main_file
->
{
security
}
eq
'
public
');
# 2020-11-23: column formatdesc in table document contains notes by UBW staff for that particular document.
# For attachments, these are all but the first document, this is stored as "description" in the attachment's metadata.
# For the main document it self, this information was not stored anywhere.
# Feature Request: store that information into utheses_note_2, e.g. by concatinating it
if
(
defined
(
$main_file
->
{
formatdesc
})
&&
$main_file
->
{
formatdesc
}
ne
'')
{
$utp
->
{
utheses_note_2
}
=
(
exists
(
$utp
->
{
utheses_note_2
}))
?
join
('
;
',
$utp
->
{
utheses_note_2
},
$main_file
->
{
formatdesc
})
:
$main_file
->
{
formatdesc
};
}
$ut
->
public
('
thesis
',
$thesis
);
# Mon May 11 22:12:38 CEST 2020 asked nd about this, especially thesis_doc_added_date:
...
...
@@ -3921,7 +3957,7 @@ mysql> select count(*), abstract_nicht_anzeigen from eprint group by abstract_ni
# diese objekte sollten wir uns notieren. bitte stattdessen einen leeren string uebergeben.
push
(
@warnings
,
{
warning
=>
'
date_app missing, can not assign assessment_date
'
});
$thesis
{
assessment_date
}
=
''
;
$thesis
{
assessment_date
}
=
undef
;
}
# print __LINE__, " thesis: ", Dumper (\%thesis);
...
...
@@ -4565,7 +4601,7 @@ sub update_policies
$db_ot2ut
=
IRMA::db::
get_any_db
(
$cnf
,
'
ot2ut_database
')
unless
(
defined
(
$db_ot2ut
));
my
$col_utp
=
$db_ot2ut
->
get_collection
('
utheses.policies
');
if
(
defined
(
$refresh_oldest_policies
)
&&
$refresh_oldest_policies
>
0
&&
$refresh_oldest_policies
<=
3
000
)
if
(
defined
(
$refresh_oldest_policies
)
&&
$refresh_oldest_policies
>
0
&&
$refresh_oldest_policies
<=
5
000
)
{
my
$c1
=
$col_utp
->
find
(
{},
{
'
_id
'
=>
1
,
eprint_id
=>
1
,
generated
=>
1
});
$c1
->
sort
(
{
generated
=>
1
}
);
...
...
@@ -4730,17 +4766,18 @@ sub policies_stats
$eprint_ids
[
$eprint_id
]
->
{
$context
}
=
$row_sync
;
if
(
defined
(
$utheses_id
))
if
(
defined
(
$utheses_id
)
&&
$error_code
eq
'
ok
'
)
{
$synced
{
$eprint_id
}
->
{
$context
}
=
[
$lastmod
,
$utheses_id
];
$totals
{
$context
}
->
{
cnt_ok
}
++
;
$blocks
[
$block_nr
]
->
{
$context
}
->
{
cnt_ok
}
++
;
$metablocks
[
$metablock_nr
]
->
{
$context
}
->
{
cnt_ok
}
++
;
}
else
{
# push (@upload_errors, $row_sync);
push
(
@
{
$upload_errors
{
$error_code
}},
$eprint_id
);
$totals
{
$context
}
->
{
cnt_error
}
++
;
$blocks
[
$block_nr
]
->
{
$context
}
->
{
cnt_error
}
++
;
$metablocks
[
$metablock_nr
]
->
{
$context
}
->
{
cnt_error
}
++
;
...
...
@@ -5123,7 +5160,7 @@ EOX
EOX
my
$block_start
=
$metablock_nr
*
100
;
my
$block_last
=
(
$metablock_nr
==
7
)
?
750
:
$block_start
+
99
;
my
$block_last
=
(
$metablock_nr
==
6
)
?
$MAX_METABLOCK
:
$block_start
+
99
;
for
(
my
$block_nr
=
$block_start
;
$block_nr
<=
$block_last
;
$block_nr
++
)
{
next
unless
(
defined
(
$blocks
[
$block_nr
]));
...
...
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment