ecaccess 4.0.1-1 / usr / share / perl5 / ECMWF / ECaccess.pm

package ECMWF::ECaccess;

use 5.008002;
use strict;
use warnings;

use SOAP::Lite;
use IO::Socket::SSL;
use Net::HTTP;
use MIME::Base64;
use URI::Escape;
use POSIX;

# For windows
use Encode::Locale;
use Encode::Byte;

require Exporter;

our @ISA     = qw(Exporter);
our $VERSION = '4.0.1';

# ECaccess Version
use constant VERSION => 'v4.0.1_2013040401';

# Force SOAP to use the IO::Socket::SSL library!
$ENV{'PERL_NET_HTTPS_SSL_SOCKET_CLASS'} = 'IO::Socket::SSL';

# Deactivate certificate verification in LWP (just in case)
$ENV{'PERL_LWP_SSL_VERIFY_HOSTNAME'} = '0';

# Force the SSLv3 protocol and de-activate the certificate verification
IO::Socket::SSL::set_ctx_defaults(
	SSL_version => 'SSLv3',
	SSL_verify_mode => 0);

# ECaccess variables

my $data_channel;
my $control_channel;

# Http/https hostnames and ports

my $http_ecaccess  = '';
my $https_ecaccess = '';

# Open ECMWF command channel to ECaccess (secured with SSL)

sub new() {

	# SOAP package to deal with reconnections!
    BEGIN {
      package My::SOAP::Lite;
      use SOAP::Lite;
      
      @My::SOAP::Lite::ISA = 'SOAP::Lite';
      my $retry = 0;
      
      sub setRetry($) {
		$retry = $_[0];
	  }

      sub call {
        my $self = shift;
        return $self->SUPER::call unless @_;
        my $result = $self->SUPER::call(@_);
      	return $result if $self->transport->is_success;
        my $try = $retry;
        # If retry is requested and the error is a transport layer error!
        while ( $try > 0 && ref($result) eq '') {
	  	  warn "Failed connection (" . $try-- . " attempt(s) remaining)\n";
	  	  sleep 5;
          $result = $self->SUPER::call(@_);
          return $result if $self->transport->is_success;
        }
	    # Get the error message
	    if ( ref($result) eq '' ) {
		  die ($self->transport->status . "\n");
	    } else {
		  die ($result->faultstring . "\n");
	    }
      }
    }

	# Get the parameters
	my $package = shift;
    my $self = {
        retry => shift,
        debug  => shift
    };
    
	# Check if debug is requested?
	if ( $self->{debug} ) {
		eval "use SOAP::Lite +trace => 'debug';";
		eval "use IO::Socket::SSL qw(debug3);";
	}

	# Set the SSL connection retry
	My::SOAP::Lite::setRetry( $self->{retry} );

	# Just in case the environment parameters are
	# not set
	no warnings qw(uninitialized);

	# Is it local?
	$http_ecaccess  = "$ENV{'http_eccmd'}";
	$https_ecaccess = "$ENV{'https_eccmd'}";

	# Get it from the environment?
	$http_ecaccess  = "$ENV{'http_ecaccess'}"  if ( $http_ecaccess  eq '' );
	$https_ecaccess = "$ENV{'https_ecaccess'}" if ( $https_ecaccess eq '' );

	# Default values if not set!
	$http_ecaccess  = 'ecaccess.ecmwf.int' if ( $http_ecaccess  eq '' );
	$https_ecaccess = 'ecaccess.ecmwf.int' if ( $https_ecaccess eq '' );

	# Init control channel
	$control_channel =
	  My::SOAP::Lite->uri('http://service.client.ecmwf/xsd')->proxy( 'https://' . $https_ecaccess . '/axis2/services/ECaccessService?wsdl', timeout => 5000 )
	  ->on_fault( sub{} );

	# Init data channel
	$data_channel =
	  My::SOAP::Lite->uri('http://service.client.ecmwf/xsd')->proxy( 'http://' . $http_ecaccess . '/axis2/services/ECaccessService?wsdl', timeout => 5000 )
	  ->on_fault( sub{} );

	return bless( {}, $package );
}

# Get the Control Channel (https)

sub getControlChannel() {
	return $control_channel;
}

# Get the Data Channel (http)

sub getDataChannel() {
	return $data_channel;
}

# Get the HOME directory (Unix or Windows)?

sub getHome() {

	# If HOME or USERPROFILE are not defined
	no warnings qw(uninitialized);

	# On Unix Platforms
	return $ENV{HOME} if $ENV{HOME};

	# On Windows Platforms
	return $ENV{USERPROFILE} if $ENV{USERPROFILE};

	# HOME is required to get/store the Certificate
	die "Please set your HOME first!\n";
}

# Get the certificate file

sub getCertificateFile() {

	# If ECCERT is not defined
	no warnings qw(uninitialized);

	# Authenticate with a Certificate file
	my $certificateFile;

	if ( $ENV{'ECCERT'} ) {

		# Defined in the ECCERT parameter
		$certificateFile = $ENV{'ECCERT'};
	}
	else {

		# By default load the certificate from HOME
		$certificateFile = ( getHome() ) . "/.eccert.crt";
	}

	return $certificateFile;
}

# Get the Security Token either from the Certificate or the eccert command when
# local to ECMWF

sub getToken() {

	# If ECCERT_HOME,HOME or ectrans_ecaccess are not defined
	no warnings qw(uninitialized);

	# The Certificate content
	my $certificateContent;

	if ( $ENV{'ECCERT_HOME'} ) {

		# Authenticate with the local eccert (ECMWF)
		my $eccert = "$ENV{'ECCERT_HOME'}/client/ecbin/" . ( POSIX::uname() )[0] . "/eccert";

		# Check if the eccert command exists and is executable
		die "No valid eccert found: " . $eccert . " (please check with your administrator)\n" if ( not( -x $eccert ) );

		# Are we using a specific eccmd port? (e.g. bucmd)
		my $ectransport_ecaccess = "$ENV{'ectransport_ecaccess'}";
		$ectransport_ecaccess = '645' if ( $ectransport_ecaccess eq '' );

		# Get the Certificate/Ticket from the eccert command
		my $command = $eccert . " -eccmdhost eccmd.ecmwf.int -eccmdport " . $ectransport_ecaccess . " -ecpass";
		$certificateContent = `$command 2>&1`;
		die "Authentication failed with eccert (please check with your administrator)\n" if not( $? >> 8 == 0 );
		chomp($certificateContent);

	}
	else {

		# Authenticate with a Certificate file
		my $certificateFile = getCertificateFile();

		# Check if the Certificate exists
		die "No valid certificate found! (please use ecaccess-certificate-create first)\n" if not( -f $certificateFile && -T $certificateFile );

		sub get_contents($) {
			my $in  = shift;
			my $ret = "";
			while (<$in>) {
				$ret .= $_;
			}
			return $ret;
		}

		# Load Certificate
		open CERT, $certificateFile or die "Error opening certificate file!\n";
		$certificateContent = get_contents( *CERT{IO} );
		close CERT;
	}

	# Login and get the Token from the Certificate
	return $control_channel->getTokenFromCertificate( encode_base64($certificateContent) )->result;
}

# Release the security token (logout)

sub releaseToken($) {
	$control_channel->releaseToken( $_[1] );
}

# Create a new ECaccess Certificate

sub getCertificate($$) {
	my $userid   = $_[1];
	my $passcode = $_[2];

	# Get the certificate file name
	my $file = getCertificateFile();

	# Login and get the certificate
	my $certificate = decode_base64( $control_channel->createCertificate( $userid, $passcode )->result );

	# Store the certificate
	open CERT, ">", $file or die "Error creating certificate file!\n";
	chmod 0600, $file;
	binmode CERT;
	print CERT $certificate;
	close CERT;
}

# Download file (REST)

sub getFileInputStream($$) {
	my $handle = $_[1];

	# Open the socket connection
	my $socket = Net::HTTP->new( Host => $http_ecaccess ) || die "Can't open data connection!\n";

	# Send the request
	$socket->write_request( GET => "/dataios?handle=" . ( uri_escape($handle) ), 'User-Agent' => "perl-ectools:" . $VERSION, 'Content-Type' => "text/xml" );

	# Read the headers
	my ( $code, $mess, %h ) = $socket->read_response_headers;

	# Return the socket
	return $socket;
}

# Upload file (REST)

sub writeFileOutputStream($$) {
	my $handle = $_[1];
	my $data   = $_[2];

	# Open the socket connection
	my $socket = Net::HTTP->new( Host => $http_ecaccess ) || die "Can't open data connection!\n";

	# Send the request
	$socket->write_request(
		POST           => "/dataios",
		'User-Agent'   => "perl-ectools:" . $VERSION,
		'Content-Type' => "multipart/form-data; boundary=---------------------------154328737501",
		"-----------------------------154328737501\r\n"
		  . 'Content-Disposition: form-data; name="fileupload"; filename="'
		  . ( uri_escape($handle) ) . '"' . "\r\n"
		  . "Content-Type: application/octet-stream\r\n\r\n"
		  . $data . "\r\n"
		  . "-----------------------------154328737501--\r\n"
	);

	# Read the headers
	my ( $code, $mess, %h ) = $socket->read_response_headers;
}

1;

__END__

# ECMWF::ECaccess Documentation

=head1 NAME

ECMWF::ECaccess - Perl extension to access the ECMWF ECaccess Web Services (or ECaccess API)

=head1 SYNOPSIS

  What are the ECaccess Web Services?
  How to use it?
    ECaccess Authentication
    ECaccess Token
    Control Channel
    Data Channel
    Upload/Download FILEs
    Release Token
    ECaccess File System
  Where can I find examples?
  See Also

=head1 DESCRIPTION

ECMWF::ECaccess is a Perl module which provides access to the ECMWF ECaccess
Web Services (or ECaccess API).

=head1 WHAT ARE THE ECACCESS WEB SERVICES?

The SOAP ECaccess API was created for developers and researchers interested in using ECMWF facilities in their applications.
Developers write software programs that connect remotely to the SOAP ECaccess Service. Communication is performed via SOAP,
an XML-based mechanism for exchanging typed information.

Developers can issue requests to the following SOAP ECaccess Methods:

=head2 Gateways Requests

  String getGatewayName();
  Boolean gatewayIsConnected();
  GatewayResponse getGateway(String token, String gateway);
  GatewayResponse[] getGatewayList(String token);

=head2 Certificates and Tokens Requests

  byte[] createCertificate(String ecuser, String passcode);
  String getTokenFromCertificate(byte[] certificate);
  String getTokenFromUserPasscode(String ecuser, String passcode);
  OperationResponse[] getOperationList(String token);
  OperationResponse getOperation(String token, String operationName);
  Boolean releaseToken(String token);

=head2 FILEs Requests

  Boolean changeFileMode(String token, Integer mode, String path);
  Boolean deleteFile(String token, String source, Boolean force);
  Boolean makeDirectory(String token, String dir);
  Boolean removeDirectory(String token, String dir);
  String getFileLastModified(String token, String source);
  Long getFileSize(String token, String source);
  Boolean copyFile(String token, String source, String target, Boolean erase);
  Boolean moveFile(String token, String source, String target);
  DirResponse[] getDirList(String token, String path, Boolean dir);
  String getInputFileHandle(String token, String source, Long offset);
  String getOutputFileHandle(String token, String target, Long offset, Integer umask);
  String getTemporaryFile(String token);

=head2 JOBs Requests

  String getJobOutputHandle(String token, String jobid);
  String getJobInputHandle(String token, String jobid);
  String getJobErrorHandle(String token, String jobid);
  JobResponse getJob(String token, String jobid);
  JobResponse[] getJobList(String token);
  Boolean deleteJob(String token, String jobid);
  String submitJob(String token, JobRequest request);
  String restartJob(String token, String jobId);
  QueueResponse[] getQueueList(String token);
  QueueDetailResponse[] getQueueDetail(String token, String queueName);

=head2 Events Requests

  EventResponse[] getEventList(String token);
  EventResponse getEvent(String token, String eventId);
  Integer sendEvent(String token, EventRequest request);

=head2 ECtrans Associations Requests

  AssociationResponse[] getAssociationList(String token,String gateway);
  AssociationResponse getAssociation(String token, String gateway, String name, Boolean template);
  Boolean putAssociation(String token, String gateway, AssociationRequest association);
  Boolean deleteAssociation(String token, String gateway, String name);

=head2 ECtrans Transfers Requests

  TransferResponse[] getTransferList(String token);
  TransferResponse getTransfer(String token, String transferId);
  Boolean deleteTransfer(String token, String transferId);
  String requestTransfer(String token, TransferRequest request);
  Boolean restartTransfer(String token, TransferRequest request, String transferId);

=head2 IOs Requests (on FILEs and JOBs transfers)

  Boolean writeStringHandle(String handle, String string);
  Boolean writeBytesHandle(String handle, byte[] bytes);
  String readStringHandle(String handle, Integer size);
  byte[] readBytesHandle(String handle, Integer size);
  Boolean closeHandle(String handle);

=head2 CosInfo Request

  String getCosInfo(String token);

The following link provides the WSDL file you can use to generate code if your environment supports it:

  http://ecaccess.ecmwf.int/axis2/services/ECaccessService?wsdl

=head1 HOW TO USE IT?

=head2 ECaccess Authentication

Using the ECMWF::ECaccess Module requires a valid certificate. Certificates
can be created with the "ecaccess-certificate-create" command from an ECMWF
user identifier and a PASSCODE (using a security token), it generates a
certificate in ".eccert.crt" in the user's home directory.

You need to ensure the following environment parameters are set with
the correct values:

  http_ecaccess=gateway.meteo.ms:9080
  https_ecaccess=gateway.meteo.ms:9443

(e.g. if your local ECaccess Gateways name is "gateway.meteo.ms" and
you are using the default ECaccess http/s ports 9080/9443)

The default values are pointing to the ecaccess.ecmwf.int server.

If you access your ECaccess Gateway through a proxy then you should
also set the following:

  http_proxy=http://proxy.meteo.ms:port
  https_proxy=https://proxy.meteo.ms:port

=head2 ECaccess Token

You should generally request a token for the majority of methods available. For
example, if your application requires submitting a new job, you should usually
request a token from your certificate which grants access to the jobs management
methods (e.g. submitJob, getJobList, deleteJob or getJobResult).

When you obtain an ECacces Token for a user, you must use that ECaccess Token
for all future interactions with the API on behalf of the user.

In order to obtain an ECaccess Token you must use the following code:

  use ECMWF::ECaccess;
  my $ecaccess = ECMWF::ECaccess->new(); # Create the ECaccess Controler
  my $token = $ecaccess->getToken(); # Get an ECaccess Token

=head2 Control Channel

Once you have an ECaccess Token you must request a Control Channel to access
the API:

  my $controlChannel = $ecaccess->getControlChannel(); # Get the Control Channel

This Control Channel will allow you to call any method of the API. For example,
let's move a FILE from ECfs to the super-computer:

  controlChannel->moveFile($token,'ec:test/a.out','c2a:/c2a/tmp/systems/syi/a.out');

Or let's get the CosInfo from ECMWF:

  print $controlChannel->getCosInfo($token)->result."\n";

=head2 Data Channel

In order to Download/Upload Data to/from ECMWF you are required to use a
Data Channel:

  my $dataChannel = $ecaccess->getDataChannel(); # Get the Data Channel

The methods available on the Data Channel are the following:

  Boolean writeStringHandle(String handle, String string);
  Boolean writeBytesHandle(String handle, byte[] bytes);
  String readStringHandle(String handle, Integer size);
  byte[] readBytesHandle(String handle, Integer size);

=head2 Upload/Download FILEs

Suppose you want to Upload a text FILE you would do the following:

  $handle = $controlChannel->getOutputFileHandle($token,'home:/test.txt',0,600)->result;
  open FILE, 'test.txt' or die $!;
  while (read(FILE, $data, 524288) > 0) {
    $dataChannel->writeStringHandle($handle,$data);
  }
  $controlChannel->closeHandle($handle);
  close FILE;

In order to Download a binary FILE you would do the following:

  $handle = $controlChannel->getInputFileHandle($token,'home:/a.out',0)->result;
  open FILE, ">", $target or die $!;
  binmode FILE;
  while (length($data = decode_base64($dataChannel->readBytesHandle($handle, 524288)->result)) > 0) {
    print FILE $data;
  }
  $controlChannel->closeHandle($handle);
  close FILE;

In both cases you get a FILE Handle through the Control Channel and then you
access the Data through the Data Channel. Once the Download/Upload is completed you
close the Handle with the 'closeHandle' method.

=head2 Release Token

Once you have finished with your session you must release your token using the following
code:

  $ecaccess->releaseToken($token); # Logout

Please note that you can also use the Control Channel to Download/Upload Data if
you require a secure connection for Data Transfers (https vs. http).

=head2 ECaccess File System

When accessing FILEs at ECMWF the following domains are available to the user logged-in:

  HOME: the $HOME directory (home:)
  SCRATCH: the $SCRATCH directory (scratch:)
  ECFS: the ECFS directory (ec:)
  ECTMP: the ECTMP directory (ectmp:)
  HOST: any server at ECMWF ({host-name}:)

The format of the path is the following: [domain:][/user-id/]path

If no user-id is specified then the current user-id is selected by default. The user-id
parameter is not valid with the HOST domain.

If no domain is specified then an absolute path will translate to an absolute path on the
ecgate server and a relative path will translate to a path in the HOME directory of the
current user.

A few examples:

  "bin/a.out"				a.out file in the $HOME/bin directory of the current user
  "home:bin/a.out"			a.out file in the $HOME/bin directory of the current user
  "/tmp/a.out"				a.out file in the /tmp directory on ecgate
  "home:/syi/bin/a.out"			a.out file in the $HOME/bin directory of user syi
  "ec:bin/a.out"			a.out file in the ECFS bin directory of the current user
  "ec:/syi/bin/a.out"			a.out file in the ECFS bin directory of user syi
  "c2a:/c2a/tmp/systems/syi/a.out"	a.out file in the /c2a/tmp/systems/syi/ directory of c2a

=head1 WHERE CAN I FIND EXAMPLES?

The scripts which are provided along with this Module are good examples of
what can be achieved with the ECMWF::ECacces Module. They use all the methods
available through the ECaccess API and show the options available as well as
the format of the messages which are return by the server.

=head1 SEE ALSO

The ECMWF::ECaccess Module is based on the SOAP-Lite Module and therefore
you might be interested in looking at the documentation related to this
Module:

  http://www.soaplite.com/

The Server Side is based on Apache Axis2 (part of the ECaccess Gateway).
Documentation is also available at the following place:

  http://ws.apache.org/axis2/

If you have questions please contact: ecaccess@ecmwf.int

To download the latest ECMWF::ECaccess Perl Module please go to:

  http://www.ecmwf.int/services/ecaccess/download/

=head1 AUTHOR

Laurent Gougeon, E<lt>Laurent.Gougeon@ecmwf.intE<gt>

=head1 COPYRIGHT AND LICENSE

Copyright (C) 2010 by ECMWF (Laurent.Gougeon@ecmwf.int)

This library is free software; you can redistribute it and/or modify
it under the same terms as Perl itself, either Perl version 5.8.8 or,
at your option, any later version of Perl 5 you may have available.

=cut