From f28837adf64c38d0ed99e79d49daa25dc8fc4667 Mon Sep 17 00:00:00 2001
From: Gerhard Gonter <ggonter@gmail.com>
Date: Sun, 6 Apr 2014 10:24:56 +0200
Subject: [PATCH] updated POD section; update of translation cache
---
lib/Redmine/DB/CTX.pm | 163 ++++++++++++++++++++++++++++++++++------
lib/Redmine/DB/MySQL.pm | 83 +++++++++++++++++---
t_sync.pl | 99 +++++++++++++++++-------
3 files changed, 287 insertions(+), 58 deletions(-)
diff --git a/lib/Redmine/DB/CTX.pm b/lib/Redmine/DB/CTX.pm
index ef9ae6a..7babcfe 100644
--- a/lib/Redmine/DB/CTX.pm
+++ b/lib/Redmine/DB/CTX.pm
@@ -1,9 +1,31 @@
+=head1 NAME
+
+ Redmine::DB::CTX;
+
+=head1 DESCRIPTION
+
+This implements what I call a "Redmine synchronisation context".
+It has a synchronisation context id (sync_conext_id), a source (src)
+and a destination (dst) which resemble database connections.
+
+=head1 SYNOPSIS
+
+ my $ctx= new Redmine::DB::CTX ('ctx_id' => $setup->{'sync_context_id'}, 'src' => $src, 'dst' => $dst);
+
+=cut
+
package Redmine::DB::CTX;
use strict;
use parent 'Redmine::DB';
+=head2 $context->sync_project ($source_project_id, $destination_project_id)
+
+sync one project
+
+=cut
+
sub sync_project
{
my $ctx= shift;
@@ -13,21 +35,54 @@ sub sync_project
$ctx->sync_project_users ($sp_id, $dp_id);
}
-=head2 $context->translate ($table_name, $src_id)
+=head1 TRANSLATION
+
+Possibly the most important aspect of a synchronisation job is the
+translation of record IDs. This is done using a translation table called
+`syncs` which is stored in the destinatios (dst) database.
+
+The "CREATE TABLE" statements can be found in the source (here) or retrieved via
+
+ my ($ddl_sync_contexts, $ddl_syncs)= Redmine::DB::CTX::get_DDL();
=cut
-sub translate
+ my $TABLE_sync_contexts= <<'EOX';
+CREATE TABLE `sync_contexts`
+(
+ `id` int(11) NOT NULL AUTO_INCREMENT,
+ `name` varchar(255) DEFAULT NULL,
+ `description` longtext DEFAULT NULL,
+ PRIMARY KEY (`id`)
+) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;
+EOX
+
+ my $TABLE_syncs= <<'EOX';
+CREATE TABLE `syncs`
+(
+ `id` int(11) NOT NULL AUTO_INCREMENT,
+ `sync_context_id` int(11) DEFAULT NULL,
+ `table_name` varchar(255) DEFAULT NULL,
+ `src_id` int(11) DEFAULT NULL,
+ `dst_id` int(11) DEFAULT NULL,
+ `sync_date` datetime DEFAULT NULL,
+ `status` int(4) DEFAULT NULL,
+ PRIMARY KEY (`id`)
+) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;
+EOX
+
+sub get_DDL { return ($TABLE_sync_contexts, $TABLE_syncs); }
+
+=head2 $context->init_translation()
+
+Read the known translation table for the give synchronisation context
+and keep it around.
+
+=cut
+
+sub init_translation
{
my $ctx= shift;
- my $table_name= shift;
- my $src_id= shift;
-
- unless (defined ($src_id))
- {
- print "TRANSLATE: table_name=[$table_name] src_id=[undef] tlt=[undef]\n";
- return undef;
- }
# fetch all known translations, they are stored in the destionation's database
my $t;
@@ -45,6 +100,29 @@ sub translate
}
}
+ return $t;
+}
+
+=head2 $context->translate ($table_name, $src_id)
+
+translate a table's id. Takes the source's id and returns the
+destination's id, if known or undef otherwise.
+
+=cut
+
+sub translate
+{
+ my $ctx= shift;
+ my $table_name= shift;
+ my $src_id= shift;
+
+ unless (defined ($src_id))
+ {
+ print "TRANSLATE: table_name=[$table_name] src_id=[undef] tlt=[undef]\n";
+ return undef;
+ }
+
+ my $t= $ctx->init_translation();
if (exists ($t->{$table_name}->{$src_id}))
{
my $x= $t->{$table_name}->{$src_id};
@@ -55,25 +133,56 @@ sub translate
return undef;
}
-sub store_translate
+=head2 $ctx->store_translation ($table_name, $src_id, $dst_id)
+
+Update translation table in the database and in the cache.
+
+=cut
+
+sub store_translation
{
my $ctx= shift;
my $table_name= shift;
my $src_id= shift;
my $dst_id= shift;
+ my $t= $ctx->init_translation();
+
my $dbh= $ctx->{'dst'}->connect();
return undef unless (defined ($dbh));
- my $ssi= "INSERT INTO syncs (sync_context_id, table_name, src_id, dst_id, sync_date, status) VALUES (?,?,?,?,now(),2)";
- print "ssi=[$ssi]\n";
- my $sth= $dbh->prepare($ssi);
- my @vals= ($ctx->{'ctx_id'}, $table_name, $src_id, $dst_id);
- print "vals: ", join (',', @vals), "\n";
- $sth->execute(@vals);
- $sth->finish();
+ # TODO: maybe we need to check if (sync_context_id, table_name, src_id)
+ # are already present in the database. Then we should update the
+ # record!
+
+ my $ssi= "INSERT INTO syncs (sync_context_id, table_name,
+ src_id, dst_id, sync_date, status) VALUES (?,?,?,?,now(),2)";
+ print "ssi=[$ssi]\n"; my $sth= $dbh->prepare($ssi); my @vals=
+ ($ctx->{'ctx_id'}, $table_name, $src_id, $dst_id); print "vals: ",
+ join (',', @vals), "\n"; $sth->execute(@vals); $sth->finish();
+
+ $t->{$table_name}->{$src_id}= [ $dst_id, undef, 2 ];
}
+=head1 USERS
+
+User-related tables are:
+
+ (handled)
+ users
+ members
+ member_roles
+
+ (not yet handled)
+ watchers (TODO: this also points to contents (tickets, wiki_pages, etc.), so this must wait)
+ user_preferences
+
+=head2 $context->sync_project_users ($source_project_id, $destination_project_id)
+
+Synchronize the users and related tables.
+
+=cut
+
sub sync_project_users
{
my $ctx= shift;
@@ -118,7 +227,7 @@ sub sync_project_users
};
$d_member_id= $dst->insert ('members', $d_member);
- $ctx->store_translate('members', $s_member_id, $d_member_id);
+ $ctx->store_translation('members', $s_member_id, $d_member_id);
}
}
@@ -200,7 +309,7 @@ inherited_from points back to member_roles.id and that record might not be synce
print "new member_role record: ", main::Dumper (\%d_mr);
$d_mr_id= $ctx->{'dst'}->insert ('member_roles', \%d_mr);
- $ctx->store_translate('member_roles', $s_mr_id, $d_mr_id);
+ $ctx->store_translation('member_roles', $s_mr_id, $d_mr_id);
}
}
@@ -219,7 +328,7 @@ sub sync_role
delete ($d_role{'id'});
my $d_role_id= $ctx->{'dst'}->insert ('roles', \%d_role);
- $ctx->store_translate('roles', $s_role_id, $d_role_id);
+ $ctx->store_translation('roles', $s_role_id, $d_role_id);
$d_role_id;
}
@@ -249,7 +358,7 @@ sub sync_user
print "cloned_user: ", main::Dumper ($d_user);
$d_user_id= $ctx->{'dst'}->insert ('users', $d_user);
- $ctx->store_translate('users', $s_user_id, $d_user_id);
+ $ctx->store_translation('users', $s_user_id, $d_user_id);
}
$d_user_id;
@@ -284,3 +393,13 @@ sync wiki
1;
+__END__
+
+=head1 TODOs
+
+=head2 statistics
+
+ We need counters about unchanged, new and updated records. Deleted records may also be necessary.
+
+=end
+
diff --git a/lib/Redmine/DB/MySQL.pm b/lib/Redmine/DB/MySQL.pm
index 7154158..ab41c10 100644
--- a/lib/Redmine/DB/MySQL.pm
+++ b/lib/Redmine/DB/MySQL.pm
@@ -9,6 +9,10 @@ use parent 'Redmine::DB';
my $show_query= 0;
my $show_fetched= 0;
+sub show_fetched { shift; $show_fetched= shift; }
+sub show_query { shift; $show_query= shift; }
+sub verbose { shift; $show_fetched= $show_query= shift; }
+
sub connect
{
my $self= shift;
@@ -46,12 +50,12 @@ sub get_all_x
# my $project= new Redmine::DB::Project (%par);
# print "project: ", main::Dumper ($project);
- my $ss = qq/SELECT * FROM $table/;
+ my $ss= "SELECT * FROM $table";
my @v= ();
if (defined ($where))
{
- print "where: ", main::Dumper ($where) if ($show_query);
+ # print "where: ", main::Dumper ($where) if ($show_query);
$ss .= ' WHERE ' . shift (@$where);
@v= @$where;
}
@@ -78,6 +82,53 @@ sub get_all_x
$t;
}
+sub insert
+{
+ my $self= shift;
+ my $table= shift;
+ my $record= shift;
+
+ my $dbh= $self->connect();
+ return undef unless (defined ($dbh));
+
+ my (@vars, @vals);
+ foreach my $an (keys %$record)
+ {
+ push (@vars, $an);
+ push (@vals, $record->{$an});
+ }
+
+ my $ssi= "INSERT INTO `$table` (". join (',', @vars) .") VALUES (" . join(',', map { '?' } @vars) . ")";
+ print "ssi=[$ssi]\n";
+ print "vals: ", join (',', @vals), "\n";
+ my $sth= $dbh->prepare($ssi);
+ $sth->execute(@vals);
+ print "ERROR: ", $dbh->errstr() if ($dbh->err);
+ $sth->finish();
+
+ return $record->{'id'} if (defined ($record->{'id'})); # id attribute was set already
+
+ my $ssq= "SELECT LAST_INSERT_ID()";
+ print "ssq=[$ssq]\n";
+ $sth= $dbh->prepare($ssq);
+ $sth->execute();
+ print "ERROR: ", $dbh->errstr() if ($dbh->err);
+ my ($id)= $sth->fetchrow_array();
+ print "INSERT: id=[$id]\n";
+
+ $id;
+}
+
+sub mysql
+{
+ my $self= shift;
+ print "self: ", main::Dumper ($self);
+
+ my @cmd= ('mysql', '-h', $self->{'host'}, '-u', $self->{'username'}, $self->{'database'}, '--password='.$self->{'password'});
+ print ">> cmd=[", join (' ', @cmd), "]\n";
+ system (@cmd);
+}
+
=head1 REDMINE STUFF
before that, this might be usable for other Rails applications
@@ -89,6 +140,26 @@ TODO: factor out...
sub get_all_projects { shift->get_all_x ('projects'); }
sub get_all_users { shift->get_all_x ('users'); }
+sub get_user
+{
+ my $self= shift;
+ my $an= shift;
+ my $av= shift;
+
+ my $res= $self->get_all_x ('users', [ $an.'=?', $av ]);
+}
+
+sub get_users
+{
+ my $self= shift;
+ my $an= shift;
+
+ # print "missing users: [", join (' ', @missing_users), "]\n";
+ my $in= $an . ' IN ('. join(',', map { '?' } @_) . ')';
+ $show_query= $show_fetched= 1;
+ $self->get_all_x ('users', [ $in, @_ ]),
+}
+
sub get_project_members
{
my $self= shift;
@@ -131,13 +202,7 @@ sub pcx_members
# last if (@missing_users > 3);
}
- if (@missing_users)
- {
- # print "missing users: [", join (' ', @missing_users), "]\n";
- my $in= 'id IN ('. join(',', map { '?' } @missing_users) . ')';
- # $show_fetched= 1;
- $self->get_all_x ('users', [ $in, @missing_users ]),
- }
+ $res->{'users'}= $self->get_users ('id', @missing_users) if (@missing_users);
$res;
}
diff --git a/t_sync.pl b/t_sync.pl
index 1dc42f1..e20708b 100755
--- a/t_sync.pl
+++ b/t_sync.pl
@@ -28,6 +28,7 @@ use Data::Dumper;
$Data::Dumper::Indent= 1;
use Redmine::DB::MySQL;
+use Redmine::DB::CTX;
# use Redmine::DB::Project; nothing here yet ...
# yeah, this should be in a config file, but
@@ -46,18 +47,24 @@ my $setup=
'config' => '/home/gg/etc/dst/database.yml',
'db' => 'production',
},
+ 'sync_context_id' => 1,
+ 'syncs' =>
+ [
+ # { 'table' => 'projects', 'src_id' => 170, 'dst_id' => 1 }
+ { 'table' => 'auth_sources', 'src_id' => 1, 'dst_id' => 1, }
+ ],
'sync_projects' =>
[
{
'src_proj' => 170,
- 'dst_proj' => 1,
+ 'dst_proj' => 2, # we could try to retrieve a translation here
}
]
};
my @parameters= ();
my $op_mode= 'usage';
-my %op_modes= map { $_ => 1 } qw(sync sdp prep);
+my %op_modes= map { $_ => 1 } qw(sync sdp prep auth mysql user syncuser);
while (my $arg= shift (@ARGV))
{
@@ -102,22 +109,69 @@ elsif ($op_mode eq 'sdp') # sdp: show destination intance's projects
my $dst_proj= $dst->get_all_projects();
print "dst_proj: ", Dumper ($dst_proj);
}
+elsif ($op_mode eq 'mysql')
+{
+ my $target= shift (@parameters);
+ usage() unless (defined ($target));
+ my $env= shift (@parameters) || 'production';
+ my $cfg= read_configs($setup, $target);
+ $cfg->mysql();
+}
+elsif ($op_mode eq 'auth')
+{
+ my $src= read_configs($setup, 'src');
+ my $dst= read_configs($setup, 'dst');
+
+ $src->show_fetched (1);
+ $src->get_all_x ('auth_sources');
+ $dst->get_all_x ('auth_sources');
+}
+elsif ($op_mode eq 'user')
+{
+ my $target= shift (@parameters);
+ usage() unless (defined ($target));
+ my $an= shift (@parameters);
+ usage() unless (defined ($an));
+ my $cfg= read_configs($setup, $target);
+
+ foreach my $av (@parameters)
+ {
+ my $user= $cfg->get_user ($an, $av);
+ print "user: ", Dumper ($user);
+ }
+
+}
+elsif ($op_mode eq 'syncuser')
+{
+ usage() unless (@parameters);
+
+ my $src= read_configs($setup, 'src');
+ my $dst= read_configs($setup, 'dst');
+ # my $x_u= $src->get_all_users();
+
+ my $ctx= new Redmine::DB::CTX ('ctx_id' => $setup->{'sync_context_id'}, 'src' => $src, 'dst' => $dst);
+ my $res= $src->get_users ('login', @parameters);
+ print "res: ", Dumper ($res);
+ foreach my $s_user_id (keys %$res)
+ {
+ $ctx->sync_user ($s_user_id, $res->{$s_user_id});
+ }
+}
elsif ($op_mode eq 'sync')
{
my $src= read_configs($setup, 'src');
my $dst= read_configs($setup, 'dst');
# my $x_u= $src->get_all_users();
+ my $ctx= new Redmine::DB::CTX ('ctx_id' => $setup->{'sync_context_id'}, 'src' => $src, 'dst' => $dst);
+
# print "setup: ", Dumper ($setup);
foreach my $sp (@{$setup->{'sync_projects'}})
{
- my $proj_id= $sp->{'src_proj'};
print "sp: ", Dumper ($sp);
- $src->pcx_members ($proj_id);
- my $pcx= $src->pcx_wiki ($proj_id);
- print "pcx: ", Dumper ($pcx);
- # print "src: ", Dumper ($src);
+
+ $ctx->sync_project ($sp->{'src_proj'}, $sp->{'dst_proj'});
}
}
elsif ($op_mode eq 'diag')
@@ -151,14 +205,14 @@ sub read_configs
my $stp= shift;
my $s= shift;
- my $ss= $stp->{$s};
+ my $ss= $stp->{$s};
- my ($yml, $db)= map { $ss->{$_} } qw(config db);
+ my ($yml, $db)= map { $ss->{$_} } qw(config db);
- print "s=[$s] yml=[$yml] db=[$db]\n";
- my $x= YAML::Syck::LoadFile ($yml);
- # $ss->{'_cfg'}=
- my $c= $x->{$db};
+ print "s=[$s] yml=[$yml] db=[$db]\n";
+ my $x= YAML::Syck::LoadFile ($yml);
+ # $ss->{'_cfg'}=
+ my $c= $x->{$db};
$c->{'adapter'}= 'mysql' if ($c->{'adapter'} eq 'mysql2');
my $m= new Redmine::DB::MySQL (%$c);
@@ -176,20 +230,11 @@ sub prepare_sync_table
my $dbh= $con->connect();
print "dbh=[$dbh]\n";
- my $ss= <<'EOX';
-CREATE TABLE `syncs`
-(
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `context_id` int(11) DEFAULT NULL,
- `table` varchar(255) DEFAULT NULL,
- `src_id` int(11) DEFAULT NULL,
- `dst_id` int(11) DEFAULT NULL,
- `sync_date` datetime DEFAULT NULL,
- PRIMARY KEY (`id`)
-) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8;
-EOX
-
+ my ($ddl_sync_contexts, $ddl_syncs)= Redmine::DB::CTX::get_DDL();
# NOTE: for some reason, this can't be sent to the database, maybe/probably I'm missing something...
- print "perform this on the database\n", "--- 8< ---\n", $ss, "--- >8 ---\n";
+ print "perform this on the database\n", "--- 8< ---\n",
+ $ddl_sync_contexts, "\n",
+ $ddl_syncs,
+ "--- >8 ---\n";
}
--
GitLab