myproxy-admin 6.1.16-1 / usr / sbin / myproxy-replicate

#!/usr/bin/perl
use strict;
use Getopt::Long;
##use File::Find;
use Fcntl ':flock'; # import LOCK_* constants
use File::stat;
##use Cwd;
##use Config;
use Data::Dumper;
use Time::Local;
use IPC::Open3;
require Pod::Usage;

my $MYPROXY_DEFAULT_PORT     = "7512";
my $MYPROXY_DEFAULT_LOCATION = "/var/lib/myproxy";
my $MYPROXY_REPLICATE_FILE   = "\.myproxy_replicate";
my $MYPROXY_DELETED_FILE     = "\.myproxy_deleted";
my $MYPROXY_PID_FILE         = "\.myproxy_pid";
my $SECONDS_PER_HOUR         = (60 * 60);

#
# Do a perl check for version >= 5.005.
#
if ( ! ( defined eval "require 5.005" ) )
{
    die "Requires at least Perl version 5.005";
}

my $gpath = $ENV{GLOBUS_LOCATION};

if (!defined($gpath))
{
  print "GLOBUS_LOCATION not defined in environment. Defaulting to /usr\n";
  $ENV{GLOBUS_LOCATION} = '/usr';
  $gpath = $ENV{GLOBUS_LOCATION};
}

# process the -option options
my ( $repository, $verbose, $debug, $help, $usage, $config );


GetOptions( 'storage|r=s' => \$repository,
            'config|c=s'  => \$config,
            'verbose|v'   => \$verbose,
            'debug|d'     => \$debug,
            'usage|u'     => \$usage,
            'help|h'      => \$help)
  or Pod::Usage::pod2usage(0);

Pod::Usage::pod2usage(0) if $help;
Pod::Usage::pod2usage(0) if $usage;

my $dbglvl = 0;

$dbglvl += 1 if( $verbose );
$dbglvl += 2 if( $debug );


my $globus_dir = $ENV{GLOBUS_LOCATION};

##
## Find the MyProxy Repository.  If one is not given check the default
## locations.
##
if( !defined($repository) )
{
  print "Checking for $MYPROXY_DEFAULT_LOCATION\n" if( $dbglvl > 0 );

  if( !(-d $MYPROXY_DEFAULT_LOCATION) )
  {
    print "Checking for $globus_dir$MYPROXY_DEFAULT_LOCATION\n" 
        if( $dbglvl > 0 );

    if( !(-d "$globus_dir$MYPROXY_DEFAULT_LOCATION") )
    {
      die "Could not find MyProxy repository in any of the default " .
          "locations.\nDefault: $MYPROXY_DEFAULT_LOCATION or " . 
          "\$GLOBUS_LOCATION$MYPROXY_DEFAULT_LOCATION.\n";
    }

    $repository = "$globus_dir$MYPROXY_DEFAULT_LOCATION";
  }
  else
  {
    $repository = $MYPROXY_DEFAULT_LOCATION;
  }

  print "Setting repository to $repository\n" if( $dbglvl > 1 );    
}

##
## Check to see if myproxy-store is found and executable
##
my $myproxy_store;
chomp($myproxy_store = `which myproxy-store 2>/dev/null`);
die "myproxy-store not in PATH, stopped" if (!(-x $myproxy_store));

##
## Check to see if myproxy-destroy is found and executable
##
my $myproxy_destroy;
chomp($myproxy_destroy = `which myproxy-destroy 2>/dev/null`);
die "myproxy-destroy not in PATH, stopped" if (!(-x $myproxy_destroy));

##
## Check for a server configuration file.  If one is not given, check the
## default locations.
##
if( !defined($config) )
{
  if( !(-e "/etc/myproxy-server.config") )
  {
    if( !(-e "$globus_dir/etc/myproxy-server.config") )
    {
      die "Could not find MyProxy configuration file in any of the " . 
          "default locations.\nDefault: /etc/myproxy-server.config or " .
          "\$GLOBUS_LOCATION/etc/myproxy-server.config.\n";
    }

    $config = "$globus_dir/etc/myproxy-server.config";
  }
  else
  {
    $config = "/etc/myproxy-server.config";
  }
}

print "Using server config file: $config\n" if( $dbglvl > 0 );

my $lst_rep_time;

my $new_rep_time = undef;
my @file_list;

my $retval = main();
exit( $retval );


###########################################################################
##  Do everything
###########################################################################

sub main
{

  ##
  ## Make sure no other instance of myproxy-replicate can run until
  ## current one finishes.
  ##
  open PID, ">$repository/$MYPROXY_PID_FILE";
  flock( PID, LOCK_EX );
  print PID "$$\n";

  my $rep_file = "$repository/$MYPROXY_REPLICATE_FILE";

  ##
  ## Get the timestamp of the last replication.
  ##
  if( -e $rep_file )
  {
    $lst_rep_time = get_last_replicate_time( $rep_file );
  }
  else
  {
    $lst_rep_time = 0;
  }

  $new_rep_time = timelocal(localtime);

  ##
  ## Retrieve all of the slave MyProxy servers from config file.
  ##
  my $slave_servers = get_slaves( $config );

  print "Slave Servers:\n" . Dumper $slave_servers if( $dbglvl > 1 );

  ##
  ## Read the repository and find the files that have changed since the
  ## last replication.
  ##
  my $files = read_dir( $repository, "\.creds" );  
  print Dumper $files if( $dbglvl > 1 );

  ##
  ## send the files to the slave servers.
  ##
  my $ret = replicate_files( $repository, $files, $slave_servers );

  if( $ret == 0 )
  {
    $ret = delete_files( $repository, $files, $slave_servers );
  }

  ##
  ## Check to see if we had a problem with either replicating or 
  ## deleting.  If there was a problem don't update .myproxy_replicate
  ## or .myproxy_delete.
  ##
  if( !$ret )
  {
    print "Replication complete: ", localtime() . "\n";
    finish_up( $repository );
  }
  else
  {
    print STDERR "Replication Failed\n";
  }


##sleep( 50 );
  flock( PID, LOCK_UN );
  close PID;
  unlink( "$repository/$MYPROXY_PID_FILE" );

  return( $ret );
}


###########################################################################
##  Functions
###########################################################################

##
## get_last_replicate_time( file )
##
## Read the replication timestamp.
##
sub get_last_replicate_time
{
  my $filename = shift;

  open LSTREP, $filename;
  $lst_rep_time = <LSTREP>;
  close LSTREP;

  return $lst_rep_time;
}

##
## get_slaves( file )
##
## Get the list of slave servers from the configuration file. 
##
sub get_slaves
{
  my $config = shift;

  my $slist; 
  my $junk;
  my @slave_list;

  open CFG, $config;

  for (<CFG>)
  {
    next if( !($_ =~ /^slave_servers/) );

    ($junk, $slist) = split /slave_servers/, $_;
    my @slaves = split /;/, $slist;

    for my $s (@slaves)
    {
      my ($server, $port) = split /:/, $s;

      chomp($server);
      chomp($port);

      my $ops = "-s $server ";
      $ops .= "-p $port " if( length($port) > 0 );

      push @slave_list, $ops;
    }
  }

  close CFG;

  return \@slave_list;
}

##
## read_dir( directory, expression )
##
## Read the MyProxy repository and find files that match expression. 
##
sub read_dir
{
  my $directory  = shift;
  my $expression = shift;

  my $files;

  my $stuff = undef;

  opendir(DIR, $directory) or die 
    print "ERROR: directory \"$directory\" could not be opened!\n";

  @file_list =
      map { $_->[0] } # Form a list of names without paths.
      grep { $_->[0] =~ /$expression/ } # extract the files.
      map { [ $_, "$directory/$_" ] } # form anonymous array [name, pathname] 
      # because readdir strips the path from the bname
      grep { ! /^\.\.?$/ } # remove the current directory and its parent
      readdir(DIR); #read all of the filenames in the directory

  for my $f (@file_list)
  {
    my $sb       = stat("$directory/$f");
    my $mod_time = localtime $sb->mtime;

    printf "File is %s, mtime %s\n", $f, $mod_time if( $dbglvl > 1 );

    if( $sb->mtime >= $lst_rep_time )
    {
      push @{$files}, $f;
    } 
  }

  $stuff->{'files'} = $files;
  
  closedir(DIR);

  $stuff->{'del'} = missing_files( $directory, @file_list );

  return $stuff;
}

##
## replicate_files( reposityr, files, slaves )
##
## Replicate all of the files listed to all of the slaves listed. 
##
sub replicate_files
{
  my $rep    = shift;
  my $files  = shift;
  my $slaves = shift;

  my ($exitstatus, $output);
  my $ret = 0;

  for my $f (@{$files->{'files'}})
  {
    my $data = $f;
    $data =~ s/creds/data/;

    my $options = parse_datafile( $rep, $data );

    print "File not found: $rep/$data\n  File $rep/$f not replicated.\n\n" 
        if( !defined($options) );
    next if( !defined($options) );

    $options .= "-c \"$rep/$f\" -y \"$rep/$f\" ";

    for my $s (@{$slaves})
    {
      print "OPTIONS: $myproxy_store\n$options $s\n\n" if( $dbglvl > 2 );

      ($exitstatus, $output) = runcmd( "$myproxy_store $options $s" );

      if( $exitstatus != 0 )
      {
        $ret = 1;
      }

      print STDERR "STATUS: $exitstatus\n $output\n" if( $dbglvl > 0 );
    }
    $options = undef;
  }

  return $ret;
}

##
## parse_datafile( repository, datafile )
##
## Read the data file and use the information in it to create the option
## list for replication. 
##
sub parse_datafile
{
  my $rep   = shift;
  my $fname = shift;

  my $options = undef;

  open( FN, "$rep/$fname" ) or return undef;

  for my $value (<FN>)
  {
    my( $tag, $val ) = split( /\=/, $value );
    # OWNER creds->owner_name
    # where does this come from?  It seems like it is something that is passed
    # but not flaged.  If this is true, how do we get it to replicate?
    if( $value =~ /OWNER=(.*)/ )
    {
    }
    # LIFETIME -t
    elsif( $value =~ /LIFETIME=(.*)/ )
    {
      $options .= "-t " . $1 / $SECONDS_PER_HOUR . " ";
    }
    # NAME -k creds->credname
    elsif( $value =~ /^NAME=(.*)/ )
    {
      $options .= "-k \"$1\" ";
    }
    # USERNAME -l 
    elsif( $value =~ /^USERNAME=(.*)/ )
    {
      $options .= "-l \"$1\" ";
    }
    # DESCRIPTION -K creds->creddesc
    elsif( $value =~ /DESCRIPTION=(.*)/ )
    {
      $options .= "-K \"$1\" ";
    }
    # RETRIEVERS -r creds->retrievers
    # what about anonymous retrievers?
    elsif( $value =~ /RETRIEVERS=(.*)/ )
    {
      $options .= "-x -r \"$1\" ";
    }
    # RENEWERS -R creds->renewers
    # what about anonymous renewers?
    elsif( $value =~ /RENEWERS=(.*)/ )
    {
      $options .= "-x -R \"$1\" ";
    }
    # KEYRETRIEVERS -E creds->keyretrieve
    elsif( $value =~ /KEYRETRIEVERS=(.*)/ )
    {
      $options .= "-x -E \"$1\" ";
    }
    # END_OPTIONS
    elsif( $tag eq "END_OPTIONS" )
    {
    }
  }
  close FN;

  return $options;
}

##
## missing_files( repository, files )
##
## Look at the last snapshot of the repository on the master.  Compare it
## to the current list of files in the repository.  If any are missing
## from the repository, they must have been deleted, so we need to delete
## them. 
##
sub missing_files
{
  my $rep   = shift;
  my @files = @_;

  my @delfiles;

  open FD, "$rep/$MYPROXY_DELETED_FILE" or return undef;

  my @fd = <FD>;

  for my $r (@fd)
  {
    my $fnd = 0;

    chomp($r);

    for my $f (@files)
    {
      chomp($f);
      $fnd = 1 if( $f eq $r );
    }

    if( !$fnd )
    {
      push @delfiles, $r;
    }
  }    

  return \@delfiles
}

##
## delete_files( repostiroy, files, slaves )
##
## Using the list of files that are to be deleted, send a destroy command
## to each of the listed slaves.
##
sub delete_files
{
  my $rep    = shift;
  my $files  = shift;
  my $slaves = shift;

  my ($exitstatus, $output);

  my $ret = 0;

  for my $f (@{$files->{'del'}})
  {
    my ($name, $ext) = split /\./, $f;
    my ($uname, $oname ) = split /-/, $name;

    my $options = "-l \"$uname\" ";
    $options .= "-k \"$oname\" " if( length($oname) > 0 );

    for my $s (@{$slaves})
    {
      print "OPTIONS: myproxy_destroy\n$options $s -v\n\n" if( $dbglvl > 2 );

      ($exitstatus, $output) = runcmd( "$myproxy_destroy $options $s -v" );

      if( ($exitstatus != 0) &&
          !($output =~ /No such file or directory/ ||
            $output =~ /do not exist/) )
      {
print "Bad delete\n";
        $ret = 1;
        # For now do nothing about this.  myproxy-server neeeds to be modified
        # to return more error information.  There is no way to tell why this
        # failed.
        #
        # There are several problems with the current retry scheme.  If we do
        # not update the date and directory snap shot.  The next time around
        # we are going to have problems.  Resending myproxy-destroy commands
        # to servers where the cred has already been destroied returns an
        # error message.  This will just cause an infinate cycle of failures.
        #
        # If we log the deletes and then rerun them we can run into problems
        # with missing creds.  If we have a case where a cred is stored but
        # the store fails and then before the next replicate, the user 
        # destories that cred we will have a problem with the destory.  The
        # server will have no idea what we are trying to destroy and return
        # an error.  Again, we end up in an infinate cycle of destories.
        #
        # If we log both the stores and destories we still have problems.  Is
        # we do a store and it fails and we log it, then the user destories
        # the cred before the next replicate.  We try and do a store on a 
        # cred that is not there.  This causes an error.  Then the destory
        # runs and we end up with another error.  Now we have two infinate
        # cycles going.
        #
        # We need a better solution!  I still like the idea of just taring
        # the directory and coping it over to each slave.
        #
      }
      print STDERR "STATUS: $exitstatus\n $output\n" if( $dbglvl > 0 );
    }
    $options = undef;
  }

  return $ret;
}

##
## runcmd( command )
##
## Run a MyProxy command and capture the exit value and the output
##
sub runcmd 
{
  my ($command) = @_;

  my $pid = open3(*Writer, *Reader, 0, "exec $command") ||
         die "failed to run $command";

  close(Writer);

  my @output = <Reader>;
  close(Reader);

  waitpid($pid, 0);

  my $exitstatus = $?;
  my $output = join('', @output);

  return ($exitstatus, $output);
}

sub write_timestamp
{
  my $rep = shift;

  open FD, ">$rep/$MYPROXY_REPLICATE_FILE";
  print FD $new_rep_time;
  close FD;
}

sub write_delete_file
{
  my $rep = shift;

  open FD, ">$rep/$MYPROXY_DELETED_FILE" or die
      print "ERROR: $rep/$MYPROXY_DELETED_FILE could not be opened!\n";

  for (@file_list)
  {
    print FD "$_\n";
  }

  close FD;
}

##
## findish_up( repository )
##
## Update the .myproxy_replicate with the current timesamp.  Update
## .myproxy_delete with the current snapshot of the repository.
##
sub finish_up
{
  my $rep = shift;

  open FD, ">$rep/$MYPROXY_REPLICATE_FILE";
  print FD $new_rep_time;
  close FD;

  open FD, ">$rep/$MYPROXY_DELETED_FILE" or die
      print "ERROR: $rep/$MYPROXY_DELETED_FILE could not be opened!\n";

  for (@file_list)
  {
    print FD "$_\n";
  }

  close FD;
}

__END__


=head1 NAME

B<myproxy-replicate> - Stores data from the MyProxy master repository to all
the slave servers.

=head1 SYNOPSIS

B<myproxy-replicate> [options] ...

  Options:
     [-verbose|-v]                      Print copious output
     [-help|-h]                         Print usage
     [-storage|-r]=<path to repository> Directory of the MyProxy repository.
     [-config|-c]=<path to config file> Directory of the MyProxy Server
                                        configuration file.
     [-debug|-d]                        Run in debug mode 

=head1 DESCRIPTION

B<myproxy-replicate> Replicates data. 

This utility will read a specified MyProxy repository and send any new or
changed data to a slave MyProxy server.  The slave servers are specified 
in the B<myproxy-server.config(5)> file.  

This utility will need to run at some specified interval in order to keep
the slave repositories semi current with the Master repository.  This can
best be accomplished using cron, or some similar mechanism.

=head1 OPTIONS

=over 8

=item B<-v>, B<-verbose>

Enables verbose debugging output to the terminal.

=item B<-h>, B<-help>

Displays command usage text and exits.

=item B<-u>, B<-usage>

Displays command usage text and exits.

=item B<-r> I<dir>, B<-storage> I<dir>

Specifies the location of the credential storage directory.
The directory must be accessible only by the user running the
B<myproxy-server> process for security reasons.  
Default: /var/lib/myproxy or $GLOBUS_LOCATION/var/myproxy

=item B<-c> I<file>, B<-config> I<file>

Specifies the location of the myproxy-server configuration file.
Default: /etc/myproxy-server.config or 
         $GLOBUS_LOCA-TION/etc/myproxy-server.config

=back

=head1 SEE ALSO

myproxy-init(1) myproxy-store(1) myproxy-retrieve(1) myproxy-delegate(1)
myproxy-server(8) myproxy-server.config(5)

=head1 AUTHOR

=cut