From b2fcc6e19572d67e84e312a953f6ed0b3a70134d Mon Sep 17 00:00:00 2001 From: Warren Usui Date: Wed, 12 Mar 2014 11:37:43 -0700 Subject: [PATCH] Remove code duplication from s3 tests. A new module, s3_utilities.pm has been created. It contains subroutines common to at least two of the workunits in this directory. Code was moved here from the other pl files, and some minor changes (paramers and scope changes) were needed. Fixes: 7472 Signed-off-by: Warren Usui --- qa/workunits/rgw/s3_bucket_quota.pl | 250 +++--------------------- qa/workunits/rgw/s3_multipart_upload.pl | 212 +------------------- qa/workunits/rgw/s3_user_quota.pl | 227 +-------------------- qa/workunits/rgw/s3_utilities.pm | 218 +++++++++++++++++++++ 4 files changed, 253 insertions(+), 654 deletions(-) create mode 100644 qa/workunits/rgw/s3_utilities.pm diff --git a/qa/workunits/rgw/s3_bucket_quota.pl b/qa/workunits/rgw/s3_bucket_quota.pl index 9292e8f284f70..7aa5a68c8e1d6 100755 --- a/qa/workunits/rgw/s3_bucket_quota.pl +++ b/qa/workunits/rgw/s3_bucket_quota.pl @@ -34,6 +34,7 @@ use IO::File; use Getopt::Long; use Digest::MD5; use Pod::Usage(); +use s3_utilities; my $help; @@ -43,133 +44,17 @@ Getopt::Long::GetOptions( Pod::Usage::pod2usage(-verbose => 1) && exit if ($help); #== local variables === -my $mytestfilename; +our $mytestfilename; my $mytestfilename1; my $logmsg; my $kruft; my $s3; my $domain = "front.sepia.ceph.com"; my $host = get_hostname(); -my $hostname = "$host.$domain:7280"; # as rgw is running on port 7280 -my $testfileloc; -my $sec; -my $min; -my $hour; -my $mon; -my $year; -my $mday; -my $wday; -my $yday; -my $isdst; -my $PASS_CNT = 0; -my $FAIL_CNT = 0; +our $hostname = "$host.$domain:7280"; # as rgw is running on port 7280 +our $testfileloc; my $rgw_user = "qa_user"; -# function to get the current time stamp from the test set up -sub get_timestamp { - ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime(time); - if ($mon < 10) { $mon = "0$mon"; } - if ($hour < 10) { $hour = "0$hour"; } - if ($min < 10) { $min = "0$min"; } - if ($sec < 10) { $sec = "0$sec"; } - $year=$year+1900; - return $year . '_' . $mon . '_' . $mday . '_' . $hour . '_' . $min . '_' . $sec; -} - -# Function to check if radosgw is already running -sub get_status { - my $service = "radosgw"; - my $cmd = "ps -ef | grep $service | grep -v grep"; - my $status = get_cmd_op($cmd); - if ($status =~ /client.radosgw/ ){ - return 0; - } - return 1; -} - -# function to execute the command and return output -sub get_cmd_op -{ - my $cmd = shift; - my $excmd = `$cmd`; - return $excmd; -} - -#Function that executes the CLI commands and returns the output of the command -sub get_command_output { - my $cmd_output = shift; - open( FH, ">>$test_log" ); - print FH "\"$cmd_output\"\n"; - my $exec_cmd = `$cmd_output 2>&1`; - print FH "$exec_cmd\n"; - close(FH); - return $exec_cmd; -} - -# Function to get the hostname -sub get_hostname -{ - my $cmd = "hostname"; - my $get_host = get_command_output($cmd); - chomp($get_host); - return($get_host); -} - -sub pass { - my ($comment) = @_; - print "Comment required." unless length $comment; - chomp $comment; - print_border2(); - print "Test case: $TC_CNT PASSED - $comment \n"; - print_border2(); - $PASS_CNT++; -} - -sub fail { - my ($comment) = @_; - print "Comment required." unless length $comment; - chomp $comment; - print_border2(); - print "Test case: $TC_CNT FAILED - $comment \n"; - print_border2(); - $FAIL_CNT++; -} - -sub print_border2 { - print "~" x 90 . "\n"; -} - -# Function to create the user "qa_user" and extract the user access_key and secret_key of the user -sub get_user_info -{ - my $cmd = "sudo radosgw-admin user create --uid=$rgw_user --display-name=$rgw_user"; - my $cmd_op = get_command_output($cmd); - if ($cmd_op !~ /keys/){ - return (0,0); - } - my @get_user = (split/,/,$cmd_op); - foreach (@get_user) { - if ($_ =~ /access_key/ ){ - $get_acc_key = $_; - } elsif ($_ =~ /secret_key/ ){ - $get_sec_key = $_; - } - } - my $access_key = $get_acc_key; - my $acc_key = (split /:/, $access_key)[1]; - $acc_key =~ s/\\//g; - $acc_key =~ s/ //g; - $acc_key =~ s/"//g; - my $secret_key = $get_sec_key; - my $sec_key = (split /:/, $secret_key)[1]; - chop($sec_key); - chop($sec_key); - $sec_key =~ s/\\//g; - $sec_key =~ s/ //g; - $sec_key =~ s/"//g; - return ($acc_key, $sec_key); -} - # Function that deletes the user $rgw_user and write to logfile. sub delete_user { @@ -184,65 +69,6 @@ sub delete_user return 0; } -# Function to get the Ceph and distro info -sub ceph_os_info -{ - my $ceph_v = get_command_output ( "ceph -v" ); - my @ceph_arr = split(" ",$ceph_v); - $ceph_v = "Ceph Version: $ceph_arr[2]"; - my $os_distro = get_command_output ( "lsb_release -d" ); - my @os_arr = split(":",$os_distro); - $os_distro = "Linux Flavor:$os_arr[1]"; - return ($ceph_v, $os_distro); -} - -# Execute the test case based on the input to the script -sub create_file { - my ($file_size) = @_; - my $cnt; - $mytestfilename = $file_size; - $testfileloc = "/tmp/".$mytestfilename; - if ($file_size == '10Mb'){ - $cnt = 1; - } elsif ($file_size == '100Mb'){ - $cnt = 10; - } elsif ($file_size == '500Mb'){ - $cnt = 50; - } elsif ($file_size == '1Gb'){ - $cnt = 100; - } elsif ($file_size == '2Gb'){ - $cnt = 200; - } - my $ret = system("dd if=/dev/zero of=$testfileloc bs=10485760 count=$cnt"); - if ($ret) { exit 1 }; - return 0; -} - -sub run_s3 -{ -# Run tests for the S3 functionality - # Modify access keys to suit the target account - our ( $access_key, $secret_key ) = get_user_info(); - if ( ($access_key) && ($secret_key) ) { - $s3 = Amazon::S3->new( - { - aws_access_key_id => $access_key, - aws_secret_access_key => $secret_key, - host => $hostname, - secure => 0, - retry => 1, - } - ); - } - -our $bucketname = 'buck_'.get_timestamp(); -# create a new bucket (the test bucket) -our $bucket = $s3->add_bucket( { bucket => $bucketname } ) - or die $s3->err. "bucket create failed\n". $s3->errstr; - print "Bucket Created: $bucketname\n"; - return 0; -} - sub quota_set_max_size { my $set_quota = `sudo radosgw-admin quota set --bucket=$bucketname --max-size=1048576000`; if ($set_quota !~ /./){ @@ -255,7 +81,7 @@ sub quota_set_max_size { } sub quota_set_max_size_zero { - run_s3(); + run_s3($rgw_user); my $set_quota = `sudo radosgw-admin quota set --bucket=$bucketname --max-size=0`; if ($set_quota !~ /./){ pass ("quota set for the bucket: $bucketname with max size as zero\n"); @@ -266,7 +92,7 @@ sub quota_set_max_size_zero { } sub quota_set_max_objs_zero { - run_s3(); + run_s3($rgw_user); my $set_quota = `sudo radosgw-admin quota set --bucket=$bucketname --max-objects=0`; if ($set_quota !~ /./){ pass ("quota set for the bucket: $bucketname with max objects as zero\n"); @@ -277,7 +103,7 @@ sub quota_set_max_objs_zero { } sub quota_set_neg_size { - run_s3(); + run_s3($rgw_user); my $set_quota = `sudo radosgw-admin quota set --bucket=$bucketname --max-size=-1`; if ($set_quota !~ /./){ pass ("quota set for the bucket: $bucketname with max size -1\n"); @@ -288,7 +114,7 @@ sub quota_set_neg_size { } sub quota_set_neg_objs { - run_s3(); + run_s3($rgw_user); my $set_quota = `sudo radosgw-admin quota set --bucket=$bucketname --max-objects=-1`; if ($set_quota !~ /./){ pass ("quota set for the bucket: $bucketname max objects -1 \n"); @@ -364,32 +190,17 @@ sub upload_file { ) and (print "upload file successful\n" ) and return 0 ) or (return 1); } -# delete keys -sub delete_keys { - (($bucket->delete_key($mytestfilename)) and return 0) or return 1; -} - # delete the bucket sub delete_bucket { #($bucket->delete_key($mytestfilename1) and print "delete keys on bucket succeeded second time\n" ) or die $s3->err . "delete keys on bucket failed second time\n" . $s3->errstr; ($bucket->delete_bucket) and (print "bucket delete succeeded \n") or die $s3->err . "delete bucket failed\n" . $s3->errstr; } -# Readd the file back to bucket -sub readd_file { - system("dd if=/dev/zero of=/tmp/10MBfile1 bs=10485760 count=1"); - $mytestfilename1 = '10MBfile1'; - print "readding file to bucket: $mytestfilename1\n"; - ((($bucket->add_key_filename( $mytestfilename1, $testfileloc, - { content_type => 'text/plain', }, - )) and (print "readding file success\n") and return 0) or (return 1)); -} - # set bucket quota with max_objects and verify sub test_max_objects { my $size = '10Mb'; create_file($size); - run_s3(); + run_s3($rgw_user); quota_set_max_obj(); quota_enable(); my $ret_value = upload_file(); @@ -399,7 +210,7 @@ sub test_max_objects { fail ( "Test max objects failed" ); } delete_user(); - delete_keys(); + delete_keys($mytestfilename); delete_bucket(); } @@ -407,7 +218,7 @@ sub test_max_objects { sub test_max_objects_per_user{ my $size = '10Mb'; create_file($size); - run_s3(); + run_s3($rgw_user); quota_set_user_objs(); quota_enable(); my $ret_value = upload_file(); @@ -417,7 +228,7 @@ sub test_max_objects_per_user{ fail ( "Test max objects for the given user failed" ); } delete_user(); - delete_keys(); + delete_keys($mytestfilename); delete_bucket(); } @@ -425,7 +236,7 @@ sub test_max_objects_per_user{ sub test_beyond_max_objs { my $size = "10Mb"; create_file($size); - run_s3(); + run_s3($rgw_user); quota_set_max_obj(); quota_enable(); upload_file(); @@ -436,7 +247,7 @@ sub test_beyond_max_objs { fail ( "set max objects and test beyond max objects failed" ); } delete_user(); - delete_keys(); + delete_keys($mytestfilename); delete_bucket(); } @@ -444,7 +255,7 @@ sub test_beyond_max_objs { sub test_beyond_max_objs_user { my $size = "10Mb"; create_file($size); - run_s3(); + run_s3($rgw_user); quota_set_user_objs(); quota_enable(); upload_file(); @@ -455,7 +266,7 @@ sub test_beyond_max_objs_user { fail ( "set max objects for a given user and test beyond max objects failed" ); } delete_user(); - delete_keys(); + delete_keys($mytestfilename); delete_bucket(); } @@ -464,13 +275,13 @@ sub test_quota_size { my $ret_value; my $size = "2Gb"; create_file($size); - run_s3(); + run_s3($rgw_user); quota_set_max_size(); quota_enable(); my $ret_value = upload_file(); if ($ret_value == 1) { pass ( "set max size and ensure that objects upload beyond max size is not entertained" ); - my $retdel = delete_keys(); + my $retdel = delete_keys($mytestfilename); if ($retdel == 0) { print "delete objects successful \n"; my $size1 = "1Gb"; @@ -486,7 +297,7 @@ sub test_quota_size { fail ( "set max size and ensure that objects beyond max size is not allowed" ); } delete_user(); - delete_keys(); + delete_keys($mytestfilename); delete_bucket(); } @@ -495,13 +306,13 @@ sub test_quota_size_user { my $ret_value; my $size = "2Gb"; create_file($size); - run_s3(); + run_s3($rgw_user); quota_set_user_size(); quota_enable(); my $ret_value = upload_file(); if ($ret_value == 1) { pass ( "set max size for a given user and ensure that objects upload beyond max size is not entertained" ); - my $retdel = delete_keys(); + my $retdel = delete_keys($mytestfilename); if ($retdel == 0) { print "delete objects successful \n"; my $size1 = "1Gb"; @@ -517,7 +328,7 @@ sub test_quota_size_user { fail ( "set max size for a given user and ensure that objects beyond max size is not allowed" ); } delete_user(); - delete_keys(); + delete_keys($mytestfilename); delete_bucket(); } @@ -526,7 +337,7 @@ sub test_quota_size_disabled { my $ret_value; my $size = "2Gb"; create_file($size); - run_s3(); + run_s3($rgw_user); quota_set_max_size(); quota_disable(); my $ret_value = upload_file(); @@ -536,7 +347,7 @@ sub test_quota_size_disabled { fail ( "bucket quota size doesnt take effect when quota is disabled" ); } delete_user(); - delete_keys(); + delete_keys($mytestfilename); delete_bucket(); } @@ -545,7 +356,7 @@ sub test_quota_size_disabled_user { my $ret_value; my $size = "2Gb"; create_file($size); - run_s3(); + run_s3($rgw_user); quota_set_user_size(); quota_disable(); my $ret_value = upload_file(); @@ -555,21 +366,12 @@ sub test_quota_size_disabled_user { fail ( "bucket quota size for a given user doesnt take effect when quota is disabled" ); } delete_user(); - delete_keys(); + delete_keys($mytestfilename); delete_bucket(); } # set bucket quota for specified user and verify -# check if rgw service is already running -sub check -{ - my $state = get_status(); - if ($state) { - exit 1; - } -} - #== Main starts here=== ceph_os_info(); test_max_objects(); diff --git a/qa/workunits/rgw/s3_multipart_upload.pl b/qa/workunits/rgw/s3_multipart_upload.pl index 681ea25c1fb45..b53232699775a 100755 --- a/qa/workunits/rgw/s3_multipart_upload.pl +++ b/qa/workunits/rgw/s3_multipart_upload.pl @@ -33,6 +33,7 @@ use IO::File; use Getopt::Long; use Digest::MD5; use Pod::Usage(); +use s3_utilities; my $help; @@ -46,201 +47,8 @@ my $s3; my $domain = "front.sepia.ceph.com"; my $host = get_hostname(); our $hostname = "$host.$domain:7280"; -my $testfileloc; -my $sec; -my $min; -my $hour; -my $mon; -my $year; -my $mday; -my $wday; -my $yday; -my $isdst; -my $PASS_CNT = 0; -my $FAIL_CNT = 0; -my $mytestfilename; -my $testfileloc; - -# function to get the current time stamp from the test set up -sub get_timestamp { - ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime(time); - if ($mon < 10) { $mon = "0$mon"; } - if ($hour < 10) { $hour = "0$hour"; } - if ($min < 10) { $min = "0$min"; } - if ($sec < 1) { $sec = "0$sec"; } - $year=$year+1900; - return $year . '_' . $mon . '_' . $mday . '_' . $hour . '_' . $min . '_' . $sec; -} - -# Function to check if radosgw is already running -sub get_status { - my $service = "radosgw"; - my $cmd = "pgrep $service"; - my $status = get_cmd_op($cmd); - if ($status =~ /\d+/ ){ - return 0; - } - return 1; -} - -# function to execute the command and return output -sub get_cmd_op -{ - my $cmd = shift; - my $excmd = `$cmd`; - return $excmd; -} - -#Function that executes the CLI commands and returns the output of the command -sub get_command_output { - my $cmd_output = shift; - open( FH, ">>$test_log" ); - print FH "\"$cmd_output\"\n"; - my $exec_cmd = `$cmd_output 2>&1`; - print FH "$exec_cmd\n"; - close(FH); - return $exec_cmd; -} - -# Function to get the hostname -sub get_hostname -{ - my $cmd = "hostname"; - my $get_host = get_command_output($cmd); - chomp($get_host); - return($get_host); -} - -sub pass { - my ($comment) = @_; - print "Comment required." unless length $comment; - chomp $comment; - print_border2(); - print "Test case: $TC_CNT PASSED - $comment \n"; - print_border2(); - $PASS_CNT++; -} - -sub fail { - my ($comment) = @_; - print "Comment required." unless length $comment; - chomp $comment; - print_border2(); - print "Test case: $TC_CNT FAILED - $comment \n"; - print_border2(); - $FAIL_CNT++; -} - -sub print_border2 { - print "~" x 90 . "\n"; -} - -# Function to create the user "qa_user" and extract the user access_key and secret_key of the user -sub get_user_info -{ - my ($rgw_user) = @_; - my $cmd = "sudo radosgw-admin user create --uid=$rgw_user --display-name=$rgw_user"; - my $cmd_op = get_command_output($cmd); - if ($cmd_op !~ /keys/){ - return (0,0); - } - my @get_user = (split/,/,$cmd_op); - foreach (@get_user) { - if ($_ =~ /access_key/ ){ - $get_acc_key = $_; - } elsif ($_ =~ /secret_key/ ){ - $get_sec_key = $_; - } - } - my $access_key = $get_acc_key; - my $acc_key = (split /:/, $access_key)[1]; - $acc_key =~ s/\\//g; - $acc_key =~ s/ //g; - $acc_key =~ s/"//g; - my $secret_key = $get_sec_key; - my $sec_key = (split /:/, $secret_key)[1]; - chop($sec_key); - chop($sec_key); - $sec_key =~ s/\\//g; - $sec_key =~ s/ //g; - $sec_key =~ s/"//g; - return ($acc_key, $sec_key); -} - -# Function that deletes the given user and all associated user data -sub purge_data -{ - my ($rgw_user) = @_; - my $cmd = "sudo radosgw-admin user rm --uid=$rgw_user --purge-data"; - my $cmd_op = get_command_output($cmd); - if ($cmd_op !~ /./){ - print "user $rgw_user deleted\n"; - } else { - print "user $rgw_user NOT deleted\n"; - return 1; - } - return 0; -} - -# Function to get the Ceph and distro info -sub ceph_os_info -{ - my $ceph_v = get_command_output ( "ceph -v" ); - my @ceph_arr = split(" ",$ceph_v); - $ceph_v = "Ceph Version: $ceph_arr[2]"; - my $os_distro = get_command_output ( "lsb_release -d" ); - my @os_arr = split(":",$os_distro); - $os_distro = "Linux Flavor:$os_arr[1]"; - return ($ceph_v, $os_distro); -} - -# Execute the test case based on the input to the script -sub create_file { - my ($file_size, $part) = @_; - my $cnt; - $mytestfilename = "$file_size.$part"; - $testfileloc = "/tmp/".$mytestfilename; - if ($file_size == '10Mb'){ - $cnt = 1; - } elsif ($file_size == '100Mb'){ - $cnt = 10; - } elsif ($file_size == '500Mb'){ - $cnt = 50; - } elsif ($file_size == '1Gb'){ - $cnt = 100; - } elsif ($file_size == '2Gb'){ - $cnt = 200; - } - my $ret = system("dd if=/dev/zero of=$testfileloc bs=10485760 count=$cnt"); - if ($ret) { exit 1 }; - return 0; -} - -sub run_s3 -{ -# Run tests for the S3 functionality - # Modify access key and secret key to suit the user account - my ($user) = @_; - our ( $access_key, $secret_key ) = get_user_info($user); - if ( ($access_key) && ($secret_key) ) { - $s3 = Amazon::S3->new( - { - aws_access_key_id => $access_key, - aws_secret_access_key => $secret_key, - host => $hostname, - secure => 0, - retry => 1, - } - ); - } - -our $bucketname = 'buck_'.get_timestamp(); -# create a new bucket (the test bucket) -our $bucket = $s3->add_bucket( { bucket => $bucketname } ) - or die $s3->err. "bucket $bucketname create failed\n". $s3->errstr; - print "Bucket Created: $bucketname \n"; - return 0; -} +our $testfileloc; +our $mytestfilename; # upload a file to the bucket sub upload_file { @@ -252,25 +60,11 @@ sub upload_file { ) and (print "upload file successful\n" ) and return 0 ) or (print "upload failed\n" and return 1); } -# delete keys -sub delete_keys { - (($bucket->delete_key($mytestfilename)) and return 0) or return 1; -} - # delete the bucket sub delete_bucket { ($bucket->delete_bucket) and (print "bucket delete succeeded \n") or die $s3->err . "delete bucket failed\n" . $s3->errstr; } -# check if rgw service is already running -sub check -{ - my $state = get_status(); - if ($state) { - exit 1; - } -} - # Function to perform multipart upload of given file size to the user bucket via s3 interface sub multipart_upload { diff --git a/qa/workunits/rgw/s3_user_quota.pl b/qa/workunits/rgw/s3_user_quota.pl index 2a72ecc8ec43f..43684a4faedaa 100755 --- a/qa/workunits/rgw/s3_user_quota.pl +++ b/qa/workunits/rgw/s3_user_quota.pl @@ -33,6 +33,7 @@ use IO::File; use Getopt::Long; use Digest::MD5; use Pod::Usage(); +use s3_utilities; my $help; @@ -42,7 +43,7 @@ Getopt::Long::GetOptions( Pod::Usage::pod2usage(-verbose => 1) && exit if ($help); #== local variables === -my $mytestfilename; +our $mytestfilename; my $mytestfilename1; my $logmsg; my $kruft; @@ -50,201 +51,9 @@ my $s3; my $domain = "front.sepia.ceph.com"; my $host = get_hostname(); our $hostname = "$host.$domain:7280"; -my $testfileloc; -my $sec; -my $min; -my $hour; -my $mon; -my $year; -my $mday; -my $wday; -my $yday; -my $isdst; -my $PASS_CNT = 0; -my $FAIL_CNT = 0; +our $testfileloc; our $cnt; -# function to get the current time stamp from the test set up -sub get_timestamp { - ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime(time); - if ($mon < 10) { $mon = "0$mon"; } - if ($hour < 10) { $hour = "0$hour"; } - if ($min < 10) { $min = "0$min"; } - if ($sec < 1) { $sec = "0$sec"; } - $year=$year+1900; - return $year . '_' . $mon . '_' . $mday . '_' . $hour . '_' . $min . '_' . $sec; -} - -# Function to check if radosgw is already running -sub get_status { - my $service = "radosgw"; - my $cmd = "pgrep $service"; - my $status = get_cmd_op($cmd); - if ($status =~ /\d+/ ){ - return 0; - } - return 1; -} - -# function to execute the command and return output -sub get_cmd_op -{ - my $cmd = shift; - my $excmd = `$cmd`; - return $excmd; -} - -#Function that executes the CLI commands and returns the output of the command -sub get_command_output { - my $cmd_output = shift; - open( FH, ">>$test_log" ); - print FH "\"$cmd_output\"\n"; - my $exec_cmd = `$cmd_output 2>&1`; - print FH "$exec_cmd\n"; - close(FH); - return $exec_cmd; -} - -# Function to get the hostname -sub get_hostname -{ - my $cmd = "hostname"; - my $get_host = get_command_output($cmd); - chomp($get_host); - return($get_host); -} - -sub pass { - my ($comment) = @_; - print "Comment required." unless length $comment; - chomp $comment; - print_border2(); - print "Test case: $TC_CNT PASSED - $comment \n"; - print_border2(); - $PASS_CNT++; -} - -sub fail { - my ($comment) = @_; - print "Comment required." unless length $comment; - chomp $comment; - print_border2(); - print "Test case: $TC_CNT FAILED - $comment \n"; - print_border2(); - $FAIL_CNT++; -} - -sub print_border2 { - print "~" x 90 . "\n"; -} - -# Function to create the user "qa_user" and extract the user access_key and secret_key of the user -sub get_user_info -{ - my ($rgw_user) = @_; - my $cmd = "sudo radosgw-admin user create --uid=$rgw_user --display-name=$rgw_user"; - my $cmd_op = get_command_output($cmd); - if ($cmd_op !~ /keys/){ - return (0,0); - } - my @get_user = (split/,/,$cmd_op); - foreach (@get_user) { - if ($_ =~ /access_key/ ){ - $get_acc_key = $_; - } elsif ($_ =~ /secret_key/ ){ - $get_sec_key = $_; - } - } - my $access_key = $get_acc_key; - my $acc_key = (split /:/, $access_key)[1]; - $acc_key =~ s/\\//g; - $acc_key =~ s/ //g; - $acc_key =~ s/"//g; - my $secret_key = $get_sec_key; - my $sec_key = (split /:/, $secret_key)[1]; - chop($sec_key); - chop($sec_key); - $sec_key =~ s/\\//g; - $sec_key =~ s/ //g; - $sec_key =~ s/"//g; - return ($acc_key, $sec_key); -} - -# Function that deletes the given user and all associated user data -sub purge_data -{ - my ($rgw_user) = @_; - my $cmd = "sudo radosgw-admin user rm --uid=$rgw_user --purge-data"; - my $cmd_op = get_command_output($cmd); - if ($cmd_op !~ /./){ - print "user $rgw_user deleted\n"; - } else { - print "user $rgw_user NOT deleted\n"; - return 1; - } - return 0; -} - -# Function to get the Ceph and distro info -sub ceph_os_info -{ - my $ceph_v = get_command_output ( "ceph -v" ); - my @ceph_arr = split(" ",$ceph_v); - $ceph_v = "Ceph Version: $ceph_arr[2]"; - my $os_distro = get_command_output ( "lsb_release -d" ); - my @os_arr = split(":",$os_distro); - $os_distro = "Linux Flavor:$os_arr[1]"; - return ($ceph_v, $os_distro); -} - -# Execute the test case based on the input to the script -sub create_file { - my ($file_size) = @_; - my $cnt; - $mytestfilename = $file_size; - $testfileloc = "/tmp/".$mytestfilename; - if ($file_size == '10Mb'){ - $cnt = 1; - } elsif ($file_size == '100Mb'){ - $cnt = 10; - } elsif ($file_size == '500Mb'){ - $cnt = 50; - } elsif ($file_size == '1Gb'){ - $cnt = 100; - } elsif ($file_size == '2Gb'){ - $cnt = 200; - } - my $ret = system("dd if=/dev/zero of=$testfileloc bs=10485760 count=$cnt"); - if ($ret) { exit 1 }; - return 0; -} - -sub run_s3 -{ -# Run tests for the S3 functionality - # Modify access keys to suit the target account - my ($user) = @_; - our ( $access_key, $secret_key ) = get_user_info($user); - if ( ($access_key) && ($secret_key) ) { - $s3 = Amazon::S3->new( - { - aws_access_key_id => $access_key, - aws_secret_access_key => $secret_key, - host => $hostname, - secure => 0, - retry => 1, - } - ); - } - -our $bucketname = 'buck_'.get_timestamp(); -# create a new bucket (the test bucket) -our $bucket = $s3->add_bucket( { bucket => $bucketname } ) - or die $s3->err. "bucket $bucketname create failed\n". $s3->errstr; - print "Bucket Created: $bucketname \n"; - return 0; -} - sub quota_set_max_size_per_user { my ($maxsize, $size1,$rgw_user) = @_; run_s3($rgw_user); @@ -264,7 +73,7 @@ sub quota_set_max_size_per_user { fail("quota set for user: $rgw_user with max_size=$maxsize failed" ); } } - delete_keys(); + delete_keys($mytestfilename); purge_data($rgw_user); return 0; } @@ -302,7 +111,7 @@ sub quota_set_max_obj_per_user { fail("quota set for user: $rgw_user with max_objects=$maxobjs failed" ); } } - delete_keys(); + delete_keys($mytestfilename); purge_data($rgw_user); } @@ -337,26 +146,11 @@ sub upload_file { ) and (print "upload file successful\n" ) and return 0 ) or (return 1); } -# delete keys -sub delete_keys { - (($bucket->delete_key($mytestfilename)) and return 0) or return 1; -} - # delete the bucket sub delete_bucket { ($bucket->delete_bucket) and (print "bucket delete succeeded \n") or die $s3->err . "delete bucket failed\n" . $s3->errstr; } -# Readd the file back to bucket -sub readd_file { - system("dd if=/dev/zero of=/tmp/10MBfile1 bs=10485760 count=1"); - $mytestfilename1 = '10MBfile1'; - print "readding file to bucket: $mytestfilename1\n"; - ((($bucket->add_key_filename( $mytestfilename1, $testfileloc, - { content_type => 'text/plain', }, - )) and (print "readding file success\n") and return 0) or (return 1)); -} - #Function to upload the given file size to bucket and verify sub test_max_objs { my ($size, $rgw_user) = @_; @@ -378,19 +172,10 @@ sub test_user_quota_max_size{ } else { pass( "user quota max size for $usr passed on $max_buckets buckets" ); } - delete_keys(); + delete_keys($mytestfilename); purge_data($usr); } -# check if rgw service is already running -sub check -{ - my $state = get_status(); - if ($state) { - exit 1; - } - } - #== Main starts here=== ceph_os_info(); check(); diff --git a/qa/workunits/rgw/s3_utilities.pm b/qa/workunits/rgw/s3_utilities.pm new file mode 100644 index 0000000000000..e0c8b7664ba4c --- /dev/null +++ b/qa/workunits/rgw/s3_utilities.pm @@ -0,0 +1,218 @@ +# Common subroutines shared by the s3 testing code +my $sec; +my $min; +my $hour; +my $mon; +my $year; +my $mday; +my $wday; +my $yday; +my $isdst; +my $PASS_CNT = 0; +my $FAIL_CNT = 0; + +# function to get the current time stamp from the test set up +sub get_timestamp { + ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime(time); + if ($mon < 10) { $mon = "0$mon"; } + if ($hour < 10) { $hour = "0$hour"; } + if ($min < 10) { $min = "0$min"; } + if ($sec < 10) { $sec = "0$sec"; } + $year=$year+1900; + return $year . '_' . $mon . '_' . $mday . '_' . $hour . '_' . $min . '_' . $sec; +} + +# Function to check if radosgw is already running +sub get_status { + my $service = "radosgw"; + my $cmd = "pgrep $service"; + my $status = get_cmd_op($cmd); + if ($status =~ /\d+/ ){ + return 0; + } + return 1; +} + +# function to execute the command and return output +sub get_cmd_op +{ + my $cmd = shift; + my $excmd = `$cmd`; + return $excmd; +} + +#Function that executes the CLI commands and returns the output of the command +sub get_command_output { + my $cmd_output = shift; + open( FH, ">>$test_log" ); + print FH "\"$cmd_output\"\n"; + my $exec_cmd = `$cmd_output 2>&1`; + print FH "$exec_cmd\n"; + close(FH); + return $exec_cmd; +} + +# Function to get the hostname +sub get_hostname +{ + my $cmd = "hostname"; + my $get_host = get_command_output($cmd); + chomp($get_host); + return($get_host); +} + +sub pass { + my ($comment) = @_; + print "Comment required." unless length $comment; + chomp $comment; + print_border2(); + print "Test case: $TC_CNT PASSED - $comment \n"; + print_border2(); + $PASS_CNT++; +} + +sub fail { + my ($comment) = @_; + print "Comment required." unless length $comment; + chomp $comment; + print_border2(); + print "Test case: $TC_CNT FAILED - $comment \n"; + print_border2(); + $FAIL_CNT++; +} + +sub print_border2 { + print "~" x 90 . "\n"; +} + +# Function to create the user "qa_user" and extract the user access_key and secret_key of the user +sub get_user_info +{ + my ($rgw_user) = @_; + my $cmd = "sudo radosgw-admin user create --uid=$rgw_user --display-name=$rgw_user"; + my $cmd_op = get_command_output($cmd); + if ($cmd_op !~ /keys/){ + return (0,0); + } + my @get_user = (split/,/,$cmd_op); + foreach (@get_user) { + if ($_ =~ /access_key/ ){ + $get_acc_key = $_; + } elsif ($_ =~ /secret_key/ ){ + $get_sec_key = $_; + } + } + my $access_key = $get_acc_key; + my $acc_key = (split /:/, $access_key)[1]; + $acc_key =~ s/\\//g; + $acc_key =~ s/ //g; + $acc_key =~ s/"//g; + my $secret_key = $get_sec_key; + my $sec_key = (split /:/, $secret_key)[1]; + chop($sec_key); + chop($sec_key); + $sec_key =~ s/\\//g; + $sec_key =~ s/ //g; + $sec_key =~ s/"//g; + return ($acc_key, $sec_key); +} + +# Function that deletes the given user and all associated user data +sub purge_data +{ + my ($rgw_user) = @_; + my $cmd = "sudo radosgw-admin user rm --uid=$rgw_user --purge-data"; + my $cmd_op = get_command_output($cmd); + if ($cmd_op !~ /./){ + print "user $rgw_user deleted\n"; + } else { + print "user $rgw_user NOT deleted\n"; + return 1; + } + return 0; +} + +# Function to get the Ceph and distro info +sub ceph_os_info +{ + my $ceph_v = get_command_output ( "ceph -v" ); + my @ceph_arr = split(" ",$ceph_v); + $ceph_v = "Ceph Version: $ceph_arr[2]"; + my $os_distro = get_command_output ( "lsb_release -d" ); + my @os_arr = split(":",$os_distro); + $os_distro = "Linux Flavor:$os_arr[1]"; + return ($ceph_v, $os_distro); +} + +# Execute the test case based on the input to the script +sub create_file { + my ($file_size, $part) = @_; + my $cnt; + $mytestfilename = "$file_size.$part"; + $testfileloc = "/tmp/".$mytestfilename; + if ($file_size == '10Mb'){ + $cnt = 1; + } elsif ($file_size == '100Mb'){ + $cnt = 10; + } elsif ($file_size == '500Mb'){ + $cnt = 50; + } elsif ($file_size == '1Gb'){ + $cnt = 100; + } elsif ($file_size == '2Gb'){ + $cnt = 200; + } + my $ret = system("dd if=/dev/zero of=$testfileloc bs=10485760 count=$cnt"); + if ($ret) { exit 1 }; + return 0; +} + +sub run_s3 +{ +# Run tests for the S3 functionality + # Modify access key and secret key to suit the user account + my ($user) = @_; + our ( $access_key, $secret_key ) = get_user_info($user); + if ( ($access_key) && ($secret_key) ) { + $s3 = Amazon::S3->new( + { + aws_access_key_id => $access_key, + aws_secret_access_key => $secret_key, + host => $hostname, + secure => 0, + retry => 1, + } + ); + } + +our $bucketname = 'buck_'.get_timestamp(); +# create a new bucket (the test bucket) +our $bucket = $s3->add_bucket( { bucket => $bucketname } ) + or die $s3->err. "bucket $bucketname create failed\n". $s3->errstr; + print "Bucket Created: $bucketname \n"; + return 0; +} + +# delete keys +sub delete_keys { + (($bucket->delete_key($_[0])) and return 0) or return 1; +} + +# Readd the file back to bucket +sub readd_file { + system("dd if=/dev/zero of=/tmp/10MBfile1 bs=10485760 count=1"); + $mytestfilename1 = '10MBfile1'; + print "readding file to bucket: $mytestfilename1\n"; + ((($bucket->add_key_filename( $mytestfilename1, $testfileloc, + { content_type => 'text/plain', }, + )) and (print "readding file success\n") and return 0) or (return 1)); +} + +# check if rgw service is already running +sub check +{ + my $state = get_status(); + if ($state) { + exit 1; + } +} +1 -- 2.39.5