#!/usr/bin/perl -w
# DSSH - Distributed Secure SHell

use strict;

# Colin Stubbs <cjstubbs@optushome.com.au> 2002
#
# Last Modified: 30/07/2003 11:19:40 AM
#
# Originally dsh <http://dsh.sourceforge.net>, see below.
# 
################################################################################
## 
## dsh - distributed shell
##
## Authors : Jason Rappleye
##           Center for Computational Research at the University at Buffalo
##           rappleye@ccr.buffalo.edu
##
##           Matthew T. Piotrowski
##           Center for Computational Research at the University at Buffalo
##           mtp22@users.sourceforge.net
## 
## Last Modified: 07/17/2001
##
## Copyright (c) 2000-2001 State University of New York at Buffalo
##
## This program is free software; you can redistribute it and/or modify
## it under the terms of the GNU General Public License as published by
## the Free Software Foundation; either version 2 of the License, or
## (at your option) any later version.
##
## This program is distributed in the hope that it will be useful,
## but WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
## GNU General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with this program; if not, write to the Free Software
## Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
##
## Modification history:
##
## 03/17/2000 jcr : initial release
##
## 07/17/2001 mtp : added parallel execution of the command, added
##    verification of the status of the nodes before execution of the command,
##    based the removal of duplicate nodes on IP addresses instead of hostnames,
##    added standard error to the node output, fixed the quoting of shell
##    metacharacters, fixed the formatting of the output when the names of nodes
##    are different lengths, added the -e switch, implemented fanout, added the
##    -t switch, added the -f switch, fixed -h switch, reorganized the code, 
##    and renamed some variables; also incorporated Dale's changes
##    
#################################################################################
#
#

use English;
use IO::Handle;
use Socket;

my %CONFIG;

$CONFIG{'PREFIX'}			= "/etc/dssh";
$CONFIG{'CONFIG_FILE'}		= "$CONFIG{'PREFIX'}/dssh.conf";
$CONFIG{'NODE_GROUPS_DIR'}	= "$CONFIG{'PREFIX'}/node_groups";
$CONFIG{'CMD_SSH'}         	= "/usr/bin/ssh";
$CONFIG{'CMD_SCP'}        	= "/usr/bin/scp";
$CONFIG{'REMOTE_PORT'}     	= getservbyname("ssh", "tcp") || "22";
$CONFIG{'REMOTE_USER'}     	= $ENV{USER} || getpwuid($REAL_USER_ID);
$CONFIG{'ALL_NODES'}       	= "ALL";
$CONFIG{'FANOUT'}          	= $ENV{FANOUT}; 
$CONFIG{'DEFAULT_TIMEOUT'} 	= 5;

my  (    
        $cmd,
        $cmdStart,
        $file,
        $group,
        $i,
        $justList,
        $longest_hostname_length,
        $node,
        $timeout,
        $force,
        $tmp_host, 
        $tmp_port,
        $tmp_file,
        $transfer_recursive,
        $retrieve_recursive,
        $transfer_destination,
        $retrieve_destination,
        $total_number_of_nodes,
		$option,
		$value,
        @cmd, 
        @node_groups, 
        @nodes,
        @transfer_names,
        @retrieve_names,
        %hostname_length,
		%node_users,
        %node_ports, 
        %node_addresses, 
        %NODE_OUTPUT, 
        %pid
    );

# Read in the config file, overrides default values
if ( -f $CONFIG{'CONFIG_FILE'} && open(FH, $CONFIG{'CONFIG_FILE'}))
    {
    while (<FH>)
        {
	chomp($_);

        if ($_ =~ /[\s\ta-zA-Z0-9]+\=[\s\ta-zA-Z0-9]+/)
            {
	    $_ =~ s/\#.*//;
	    $_ =~ s/^[\s\t]+//;
	    $_ =~ s/[\s\t]+$//;
	    $_ =~ s/[\s\t]+\=[\s\t]+/\=/;

		($option, $value) = split(/\=/, $_);

		if (defined($CONFIG{$option}))
			{ $CONFIG{$option} = $value; }
            }
        }
    }

$transfer_recursive=0;
$retrieve_recursive=0;

&process_command_line_switches;
&process_remaining_args;
&check_that_there_are_node_groups_or_nodes_to_work_with;
&expand_node_groups_to_nodes;
&remove_duplicate_nodes;
&check_that_we_can_ssh_to_nodes;
&find_length_of_longest_node_name;
&process_commands;
exit;

sub process_command_line_switches 
    {
    if ($#ARGV == -1 && ! defined $ENV{'WCOLL'}) 
        {
        print STDOUT "$0: no nodes specified, check the dssh man page for the 4 ways to specify nodes\n";
        &usage;
        exit(1);
        }
    for ($i = 0; $i <= $#ARGV; $i++) 
        {
        if ($ARGV[$i] eq "-N") 
                {
                do {
                    if ($ARGV[++$i] =~ /^\-/) 
                        {
                        print STDOUT "$0: invalid nodegroup \"$ARGV[$i]\" specified ";
                        print STDOUT "with -N switch\n";
                        &usage;
                        exit(1);
                        }
                    foreach (split(",", $ARGV[$i]))
                        { push @node_groups, "$CONFIG{NODE_GROUPS_DIR}/$_"; }
                } while ($ARGV[$i] =~ /,$/);
                }
            elsif ($ARGV[$i] eq "-a") 
                { push @node_groups, "$CONFIG{NODE_GROUPS_DIR}/$CONFIG{ALL_NODES}"; }
            elsif ($ARGV[$i] eq "-p") 
                {
                if ($ARGV[++$i] !~ /^[0-9]*$/)                     {
                    print STDOUT "$0: invalid port number \"$ARGV[$i]\" specified with -p switch\n";
                    &usage;
                    exit(1);
                    }
                $CONFIG{REMOTE_PORT} = $ARGV[$i];
                }
            elsif ($ARGV[$i] eq "-u") 
                {
                if ($ARGV[++$i] =~ /^\-/)
                    {
                    print STDOUT "$0: invalid username \"$ARGV[$i]\" supplied after -u switch\n";
                    &usage;
                    exit(-1);
                    }
                $CONFIG{REMOTE_USER} = $ARGV[$i];
                }
            elsif ($ARGV[$i] eq "-w") 
                {
                do {
                    if ($ARGV[++$i] =~ /^\-/) 
                        {
                        print STDOUT "$0: invalid node \"$ARGV[$i]\" specified with -w switch\n";
                        &usage;
                        exit(1);
                        }
                    push @nodes, split(",", $ARGV[$i]);
                } while ($ARGV[$i] =~ /,$/);
                }
            elsif ($ARGV[$i] eq "-q") 
                { $justList = 1; }
            elsif ($ARGV[$i] eq "-e") 
                {
                if ($ARGV[++$i] =~ /^\-/) 
                    {
                    print STDOUT "$0: invalid command \"$ARGV[$i]\" specified ";
                    print STDOUT "with -e switch\n";
                    &usage;
                    exit(1);
                    }
                $cmd = $ARGV[$i];
                }
            elsif ($ARGV[$i] eq "-t") 
                {
                if ($ARGV[++$i] !~ /^-?\d+$/) 
                    {
                    print STDOUT "$0: invalid timeout \"$ARGV[$i]\" specified with -t switch\n";
                    &usage;
                    exit(1);
                    }
                $timeout = $ARGV[$i];
                }
            elsif ($ARGV[$i] eq "-tr")
                { $transfer_recursive=1; }
            elsif ($ARGV[$i] eq "-tf")
                {
                do {
                    if ($ARGV[++$i] =~ /^\-/) 
                        {
                        print STDOUT "$0: invalid node \"$ARGV[$i]\" specified with -tf switch\n";
                        &usage;
                        exit(1);
                        }
                    push @transfer_names, split(",", $ARGV[$i]);
                } while ($ARGV[$i] =~ /,$/);
                }
            elsif ($ARGV[$i] eq "-td")
                {
                if ($ARGV[++$i] =~ /^\-/)
                    {
                    print STDOUT "$0: invalid directory name \"$ARGV[$i]\" supplied after -td switch\n";
                    &usage;
                    exit(-1);
                    }
                else
                    { $transfer_destination=$ARGV[$i]; }
                }
            elsif ($ARGV[$i] eq "-rr")
                { $retrieve_recursive=1; }
            elsif ($ARGV[$i] eq "-rf")
                {
                do {
                    if ($ARGV[++$i] =~ /^\-/) 
                        {
                        print STDOUT "$0: invalid node \"$ARGV[$i]\" specified with -rf switch\n";
                        &usage;
                        exit(1);
                        }
                    push @retrieve_names, split(",", $ARGV[$i]);
                } while ($ARGV[$i] =~ /,$/);
                }
            elsif ($ARGV[$i] eq "-rd")
                {
                if ($ARGV[++$i] =~ /^\-/)
                    {
                    print STDOUT "$0: invalid directory name \"$ARGV[$i]\" supplied after -rd switch\n";
                    &usage;
                    exit(-1);
                    }
                else
                    { $retrieve_destination=$ARGV[$i]; }
                }
	    elsif ($ARGV[$i] eq "-sn" && $CONFIG{CMD_SSH} !~ /StrictHostKeyChecking/)
		{ 
		$CONFIG{'CMD_SSH'} .= ' -o "StrictHostKeyChecking no"';
		$CONFIG{'CMD_SCP'} .= ' -o "StrictHostKeyChecking no"';
		}
	    elsif ($ARGV[$i] eq "-sa" && $CONFIG{CMD_SSH} !~ /StrictHostKeyChecking/)
	        {
		$CONFIG{'CMD_SSH'} .= ' -o "StrictHostKeyChecking ask"';
		$CONFIG{'CMD_SCP'} .= ' -o "StrictHostKeyChecking ask"';
		}
            elsif ($ARGV[$i] eq "-sy" && $CONFIG{CMD_SSH} !~ /StrictHostKeyChecking/)
	        {
		$CONFIG{'CMD_SSH'} .= ' -o "StrictHostKeyChecking yes"';
		$CONFIG{'CMD_SCP'} .= ' -o "StrictHostKeyChecking yes"';
		}
            elsif ($ARGV[$i] eq "-f")
                { $force = 1; }
            elsif ($ARGV[$i] eq "-h") 
                {
                &usage;
                exit;
                }
	    elsif ($ARGV[$i] eq "-W")
	        {
		$CONFIG{'CMD_SSH'} .= ' -q';
		$CONFIG{'CMD_SCP'} .= ' -q';
		}
            elsif ($ARGV[$i] eq "-F")
	        {
		if ($ARGV[++$i] =~ /^[0-9]+$/)
			{ $CONFIG{'FANOUT'} = $ARGV[$i]; }
		}
            elsif ($ARGV[$i] eq "-fu")
	        {
		# why on earth would anyone have a username starting with a - ?
		if ($ARGV[++$i] !~ /^-.*/)
			{ $CONFIG{'FORCE_USER'} = $ARGV[$i]; }
		else
			{ 
			print STDOUT "Sorry, I consider usernames starting with '-' to be illegal\n"; 
			exit(-1);
			}
		}
        else
            {
            print STDOUT "$0: invalid switch \"$ARGV[$i]\":\n";
            &usage;
            exit(1);
            }
        }
    }

sub process_remaining_args 
    {
    $cmdStart = $i;
    push @cmd, @ARGV[$cmdStart..$#ARGV];
    $cmd ||= join(" ", @cmd);
    $cmd = "\'" . $cmd . "\'";
    }

sub check_that_there_are_node_groups_or_nodes_to_work_with 
    {
    if ((! @node_groups && ! @nodes)) 
        {
        if (defined($ENV{WCOLL})) 
            { push @node_groups, $ENV{"WCOLL"}; }
        else 
            {
            print STDOUT "$0: no nodes specified, check the dssh man page for the 4 ways to specify nodes\n";
            exit(1);
            }
        }
    }

sub expand_node_groups_to_nodes 
    {
    foreach $group (@node_groups) 
        {
        open (FILE_IN, "< $group") or print STDOUT "$0: couldn't open file \"$group\" : $!\n" and exit(1);
        while (<FILE_IN>) 
            {
            chomp;
            push @nodes, $_ unless (/^\#/ || /^\s*$/);
            }
        close FILE_IN;
        }
    }

sub remove_duplicate_transfer_names
    {
    my %names_exist;
    my $use_file;

    @transfer_names = grep 
        {
        $use_file = 0;
        if ( -e $_ )
            {
            if (!$names_exist{$_})
                { $use_file=1; }
            }
        $use_file == 1;
        } @transfer_names;
    }

sub remove_duplicate_retrieve_names
    {
    my %names_exist;
    my $use_file;

    @retrieve_names = grep
        {
        $use_file = 0;
        if (!$names_exist{$_})
            { $use_file=1; }
        $use_file == 1;
        } @retrieve_names;
    }

sub remove_duplicate_nodes 
    {
    my %nodes_seen;
    my @resolved_ips;
    my @name_lookup;
    my $use_node;
    my $tmp_var;
	my $tmp_user;
    @nodes = grep 
        {
	if ($_ =~ /\@/)
		{ ($tmp_user, $_) = split(/\@/, $_); }
	else
		{ $tmp_user = $CONFIG{'REMOTE_USER'}; }

        if ($_ =~ /\:/)
            { ($tmp_host, $tmp_port) = split(/\:/, $_); }
        else
            { $tmp_host=$_; $tmp_port=$CONFIG{'REMOTE_PORT'}; }

        @name_lookup = gethostbyname($tmp_host) 
        or print STDOUT "$0: could not lookup hostname \"$tmp_host\":  $!\n" and exit(1);
        @name_lookup = @name_lookup[4 .. $#name_lookup];
        @resolved_ips = map {inet_ntoa($_)} @name_lookup;
        $node_addresses{$_} = $resolved_ips[0];

	if (defined($CONFIG{'FORCE_USER'}))
		{ $node_users{$_} = $CONFIG{'FORCE_USER'}; }
	else
		{ $node_users{$_} = $tmp_user; }

        $node_ports{$_} = $tmp_port;

        $use_node=0;
        $tmp_var="";

        foreach my $ip (@resolved_ips) 
            {
            $tmp_var="$ip:$tmp_port";
            if (!$nodes_seen{$tmp_var})
                { $use_node=1; }
		    }
        $use_node == 1;
        } @nodes;
    }

sub check_that_we_can_ssh_to_nodes 
    {
    my $current_node_index = 0;
    while ($current_node_index < @nodes) 
        {
        unless (we_can_ssh_to($nodes[$current_node_index])) 
            {
            unless ($force) 
                {
                my $answer = "";
                until ($answer =~ /^y$/i || $answer =~ /^n$/i) 
                    {
                    print STDOUT "The command hasn't been run on any nodes yet, would you like to continue?\n [y or n] ";
                    $answer = <STDIN>;
                    chomp($answer);
                    }
                exit(1) unless ($answer =~ /^y$/i);
                }
            splice(@nodes, $current_node_index, 1);
            next;
            }
        $current_node_index++;
        }
    if (@nodes == 0) 
        {
        print STDOUT "$0: Can't reach any of the specified nodes. Exiting...\n";
        exit(1);
        }
    }

sub we_can_ssh_to 
    {
    my $node = shift;
    my $node_ip_address = $node_addresses{$node};
    my $node_port_number = $node_ports{$node};
    my $socket_structure = sockaddr_in($node_port_number, inet_aton($node_ip_address));

    my $proto = getprotobyname('tcp');
    socket(SOCKET_HANDLE, PF_INET, SOCK_STREAM, $proto);

    local $SIG{ALRM} = sub
        {
        print STDOUT "$0: couldn't connect to $node_users{$node}\@$node_addresses{$node}:$node_ports{$node}: \n";
        print STDOUT "$0: It appears the node is down or unreachable.\n";
        die "connection timed out\n";
        };
    $timeout = $CONFIG{'DEFAULT_TIMEOUT'} unless defined($timeout);
    if ($timeout < 0) 
        {
        print STDOUT "$0: $timeout is not a valid timeout value.\n";
        print STDOUT "$0: using default timeout value ($CONFIG{'DEFAULT_TIMEOUT'})\n";
        $timeout = $CONFIG{'DEFAULT_TIMEOUT'};
        }
    alarm($timeout);
    eval 
        {
        unless (defined (connect(SOCKET_HANDLE, $socket_structure))) 
            {
            print STDOUT "$0: couldn't connect to $node_users{$node}\@$node_addresses{$node}:$node_ports{$node}: \n";
            print STDOUT "$0: It appears the node is up but isn't allowing incoming ssh connections.\n";
            die "connection refused\n";
            }
        };
    close(SOCKET_HANDLE);
    alarm(0);
    if ($@) 
        { return undef; }
    else 
        { return 1; }
    }

sub find_length_of_longest_node_name 
    {
    $longest_hostname_length = 0;
    foreach $node(@nodes) 
        {
        my $current_hostname_length = length "$node_users{$node}\@$node_addresses{$node}\:$node_ports{$node}";
        if ($current_hostname_length > $longest_hostname_length) 
            { $longest_hostname_length = $current_hostname_length; }
        $hostname_length{$node} = $current_hostname_length;
        }
    }

sub process_commands
    {
    my $scp_command = "";
    if ($justList) 
        {
        print join("\n", @nodes), "\n";
        exit(0);
        }
    
    &pre_exec_setup;
    if (@transfer_names)
        {
        &remove_duplicate_transfer_names;
        &run_transfer_in_serial; 
        }
    &pre_exec_setup;
    if (@retrieve_names)
        {
        &remove_duplicate_retrieve_names;
        &run_retrieve_in_serial;
        }
    if ($cmd && ($cmd ne '\'\''))
        { 
        &pre_exec_setup;
        &run_cmd_in_parallel(); 
        }
    }

sub pre_exec_setup
    {
    $total_number_of_nodes = @nodes;
    if (not defined($retrieve_destination))
        { $retrieve_destination = $ENV{HOME}; }
    if (defined($CONFIG{'FANOUT'})) 
        {
        if ($CONFIG{'FANOUT'} < 1) 
            {
            print STDOUT "$0: $CONFIG{FANOUT} is not a valid fanout value:\n";
            print STDOUT "$0: using closest valid fanout value (1)\n";
            $CONFIG{'FANOUT'} = 1;
            }
        if ($CONFIG{'FANOUT'} > $total_number_of_nodes) 
            {
            print STDOUT "$0: the fanout value specified ($CONFIG{FANOUT}) is larger than the number of nodes:\n";
            print STDOUT "$0: using closest valid fanout value ($total_number_of_nodes)\n";
            $CONFIG{'FANOUT'} = $total_number_of_nodes;
            }
        }
    else 
        { $CONFIG{'FANOUT'} = $total_number_of_nodes; }
    select STDOUT;
    $| = 1;
    $| = 0;
    select STDOUT;
    $| = 1;
    $| = 0;
    }

# WORK

sub run_retrieve_in_parallel
    { }

sub run_transfer_in_parallel
    { }

# END WORK

sub run_retrieve_in_serial
    {
    my $sys_exec;

    print STDOUT "serial retrieval of files:";
    foreach $file (@retrieve_names)
        { print STDOUT " $file"; }
    print STDOUT "\n";

    for (my $starting_node = 0; $starting_node < $total_number_of_nodes; $starting_node += $CONFIG{'FANOUT'})
        {
        my $ending_node = $starting_node + $CONFIG{'FANOUT'} - 1;

        if ($ending_node >= $total_number_of_nodes)
            { $ending_node = $#nodes; }

        foreach my $node (@nodes[$starting_node..$ending_node])
            {
	    my($base_destination) = "$retrieve_destination/$node_users{$node}" . "_" . $node_addresses{$node} . "_" . $node_ports{$node};

            if (! -e $base_destination)
                { mkdir($base_destination, 0755); }
            elsif (! -d $base_destination)
                { 
                print STDOUT "Error: Retrieve destination '$base_destination' exists and is not a directory\n";
                exit(-1);
                }

            $sys_exec = "$CONFIG{'CMD_SCP'} -C -P $node_ports{$node}";

            if ($retrieve_recursive == 1)
                { $sys_exec = "$sys_exec -r"; }

            print STDOUT "[$node]\n";

            foreach $file (@retrieve_names)
                {
				my($path, $target) = ("", "");

				if ($file =~ /^\/.*\/.*/)
					{
					$path = $file;
					$path =~ s/^\///;
					$path =~ s/\/[a-zA-Z0-9\.\-\_]+$//;
					}

				$target = $file;
				$target =~ s/^.*\/([a-z0-9\.-]*)$/$1/;

				if (! -e "$base_destination/$path")
					{
					# FIX so we don't rely on system mkdir
					# @tmp_arry = split(/\//, $path);

					system("mkdir -p $base_destination/$path"); 
					}
				else
					{
					# neither the goose nor the gander fly ...
					if (! -d "$base_destination/$path")
						{ 
						print STDOUT "I expect $base_destination/$path to be a directory"; 

						exit(-1);
						}
					}

				$sys_exec = "$sys_exec $node_users{$node}\@$node_addresses{$node}:$file $base_destination/$path/";

		 		system($sys_exec);
	    	    }
            }
        }
    }

sub run_transfer_in_serial
    {
    my $sys_exec;

    print STDOUT "serial transfer of files:";
	
    foreach $file (@transfer_names)
        { print STDOUT " $file"; }
    print STDOUT "\n";

    for (my $starting_node = 0; $starting_node < $total_number_of_nodes; $starting_node += $CONFIG{'FANOUT'}) 
        {
        my $ending_node = $starting_node + $CONFIG{'FANOUT'} - 1;

        if ($ending_node >= $total_number_of_nodes) 
            { $ending_node = $#nodes; }

	my($tmp_destination);

        foreach my $node (@nodes[$starting_node..$ending_node]) 
            {                 
            $sys_exec = "$CONFIG{'CMD_SCP'} -C -P $node_ports{$node}";

            if ($transfer_recursive==1)
                { $sys_exec = "$sys_exec -r"; }

            print STDOUT "[$node_users{$node}\@$node_addresses{$node}:$node_ports{$node}]\n";

            foreach $file (@transfer_names)
                { $sys_exec = "$sys_exec $file"; }

	    	if (! defined($transfer_destination))
				{ $tmp_destination = "/home/$node_users{$node}"; }
	    	else
	        	{ $tmp_destination = $transfer_destination; }

            $sys_exec = "$sys_exec $node_users{$node}\@$node_addresses{$node}:$tmp_destination 2>&1";

            system($sys_exec);
            }
        }
    }

sub run_cmd_in_parallel 
    {
    print STDOUT "executing $cmd\n";
    for (my $starting_node = 0; $starting_node < $total_number_of_nodes; $starting_node += $CONFIG{'FANOUT'}) 
        {
        my $ending_node = $starting_node + $CONFIG{'FANOUT'} - 1;
        if ($ending_node >= $total_number_of_nodes) 
            { $ending_node = $#nodes; }
        foreach my $node (@nodes[$starting_node..$ending_node]) 
            {
            FORK: {
                $NODE_OUTPUT{$node} = new IO::Handle;
                if ($pid{$node} = open($NODE_OUTPUT{$node}, "-|")) 
                    {}
                elsif (defined $pid{$node}) 
                    {
                    exec "$CONFIG{'CMD_SSH'} -l $node_users{$node} -p $node_ports{$node} $node_addresses{$node} $cmd 2>&1" 
                    or print STDOUT "couldn't $CONFIG{'CMD_SSH'} to this node: $!\n"
                    and exit;
                    }
                }
            }
        foreach $node (@nodes[$starting_node..$ending_node]) 
            {
            my $print_padding = " " x ($longest_hostname_length - $hostname_length{$node});
            my $NODE_OUTPUT = $NODE_OUTPUT{$node};
            my $count = 0;
            while (<$NODE_OUTPUT>)
                { print STDOUT $node_users{$node} . "@" . "$node_addresses{$node}:$node_ports{$node}|$print_padding \t", $_; $count++; }
            close($NODE_OUTPUT);
            if ($count == 0)
                { print STDOUT $node_users{$node} . "@" . "$node_addresses{$node}:$node_ports{$node}|$print_padding \tNULL\n"; }
            }
        }
    }

sub usage     {
    print STDOUT << "USAGE";
usage: $0
    -a adds all nodes in the file $CONFIG{'NODE_GROUPS_DIR'}/ALL
    -e 'command'
       executes the command on all the nodes
    -f if this flag is specified, dssh won't prompt the user whether or not to
       continue if a node is unreachable or refusing a remote connection
    -F <number of nodes>
       set the fanout level
    -h display this information
    -N group1, group2, ... 
       adds the nodes in the files $CONFIG{'NODE_GROUPS_DIR'}/group1,
       $CONFIG{'NODE_GROUPS_DIR'}/group2, etc.
    -p port
       use non-standard port number for connection
    -q lists the nodes where dssh would execute the command without actually
       executing the command
    -tf file1, file2, ...., fileN
       Transfer files to remote system
    -tr Do recursive file and directory transfer
    -td directory
       Place sending files in remote destination directory
    -rf file1, file2, ...., fileN
       Retrieve files from remote system
    -rr Do recrusive file and directory retrieval    
    -rd directory
       Place retrieved files in local destination directory
    -sn Specifies you do not want strict host key checking
    -sa Specifies you want to be asked about changed or new host keys
    -sy Specifies you do not want to be asked, and will not accept connection
        to a host with a changed host key
    -t time_in_seconds 
       specifies the time to wait for a node to respond before labelling it 
       "unreachable"
    -u user
       login as a specific user on the remote host
    -fu user
       force login as a specific user on the remote host, overrides any user 
       set in a node group file for that host
    -w node1, node2, ... 
       adds the nodes node1, node2, etc.
    -W 
       Causes all warning and diagnostic messages to be suppressed

USAGE
    }

# EOF

