#!/usr/bin/perl -w # MirrorUp_genius_1.1.0.pl use strict; use Time::localtime; use Data::Dumper; my $VERSION='1.1.0'; my $source_dir_path; my $debug=0; $|=1; =information Author : Sanjoga Sahu. Date of Modification : 29th, April, 2009. (v1.0.0) 28th, March, 2010. (v1.1.0) [Modified for Latest Linux [fedora Core 12]] Operating System(s) : Any Unix/Linux Execution Method : Keep the Script in the working directory or set the path to which you need to take backup/Mirror. Run the script Manually or set crontab or .profile for daily or sheduled execution. Description : Takes Backup/Mirror of a Given directory; Also Keeps Versions of files and directories Detailed Process Info : Takes Backup/Mirror by Creating following Directories: i. /{MIRROR_LOC_path}/MirrorUp ii. /{MIRROR_LOC_path}/MirrorUp/MirrorUp_{MirrorUp_dirname}_stable Morethan [30 days old (default)] files/directories stored in this directory. iii. /{MirrorUp_LOC_path}/MirrorUp/MirrorUp_{MirrorUp_dirname}_latest Currently Modified Files and Directories of Lessthan [30 days Old (default)] stored in this directory. iv. /{MirrorUp_LOC_path}/MirrorUp/MirrorUp_logs. Keeps All MirrorUp process logs. USAGE: MirrorUp_genius_v2.pl SOURCE_LOC=/path/dirsource MirrorUp_LOC=/path/dirdest VER=10 CHECK_DATE=NO Ex: ./MirrorUp_genius_v2.pl SOURCE_LOC=/home/ssahu/dev_san/ MirrorUp_LOC=/home/ssahu/MirrorUp Setting of Command line Options: (Default variables) ---------------------------------------------------------------------- 1. SOURCE_LOC : Specify the absolute path of the Directory to MirrorUp. Default : pwd [current working directory] Ex: SOURCE_LOC=/home/abc/dev Script Variable of "SOURCE_LOC" ($source_dir_path=pwd) 2. MIRRORUP_LOC : Specify the absolute path of directory name where You like to keep the MirrorUp. Default : $ENV{'HOME'}/MirrorUp [User's home directory and creates a Directory "MirrorUp"] Script Variable of "MirrorUp_LOC" ($MirrorUp_dir_location="$ENV{'HOME'}/MirrorUp") 3. VER : Specify Number of Versions of files to keep. Default : VER=10 Ex: VER=10 (keeps 10 versions of a file, if versions reached to the threshold value the oldest version removed. Script Variable of "VER" ($cut_off_ver=10). 4. CHECK_DATE : Specify option whether to check the script to avoid multiple MirrorUp process on the same date or not. Default : NO [or 0] Ex: CHECK_DATE=YES [or 1] (Avoid multiple MirrorUp process on the same date) Ex: CHECK_DATE=NO (Takes the MirrorUp as and when the script is executed) Script Variable of "CHECK_DATE" ($current_date_check=0) 5. STABLE_CUTOFF : Specify the Number of days old to see to discriminate the file/directory as Stable Default : 30 Ex: STABLE_CUTOFF=60 Script Variable of "STABLE_CUTOFF" ($stable_cutoff=30) 6. -h : Shows Usage 7. -v : Shows Version of Script 8. i : Shows The Summary of Configuration parameters and Asks interactily to go Ahead. (default) Default : ON [ Or 1] Set i=OFF [Or 0] Script Variable of "i" ($interactive=1) ---------------------------------------------------------------------- ADDITIONAL INFORMATION AND HELP: Crontab : You may set the crontab to take a MirrorUp every day. Example: 00 23 * * * $HOME/path_of_script/MirrorUp_genius_1.0.0.pl >> $HOME/MirrorUp.out 2>&1 ---------------------------------------------------------------------- =cut my $process_flg=0; my $date=`date`; chomp $date; ##################################################################### ######################### DEFAULT VARIABLES ######################### ##################################################################### #Runs the Script in Interactive mode #Default is 1 my $interactive=1; ###################### Number of Versions keep ###################### #The Maximum Number of Versions of a Directory/file to keep in the MirrorUp Directory. # if the file or directory has been changed then the MirrorUp script is able to # distingush the latest file. so latest file along with the old versions are # kept in the MirrorUp directory. my $cut_off_ver=50; ##################### Location of Directory to be MirrorUped ######### #Default : The scrpt would take MirrorUp of current working directory. # Uncomment this variable to set the path of directory to be MirrorUped. my $pwd=`pwd`; chomp $pwd; $source_dir_path=$pwd; #$source_dir_path="$ENV{'HOME'}/dev_san"; ##################### Location of MirrorUp Directory ################## # Default : One Level Up (Parent Directory) to the Directory to be MirrorUped. my $bak_dir=$ENV{'HOME'}; my $MirrorUp_dir_location=$bak_dir."/MirrorUp"; #my $MirrorUp_dir_location="$ENV{'HOME'}/MirrorUp_Test/MirrorUp"; ##################### Check for Current Date MirrorUp and Exit ######## #set the variable (1) -> Check for the MirrorUp log file of today's date and if found one exit. (deny Multople MirrorUp on same date) #set the variable (0) -> Ignores MirrorUp log file of today's date and takes latest MirrorUp. (Allow Multiple MirrorUp on same date) my $current_date_check=0; ##################################################################### #Default : Stable -> to get 1 month old file /directory my $stable_cutoff=30; ##################################################################### # Settting Up Configuration through command line Arguments. my $back; my $help=0; my $c_ver=0; my $ask="YES"; my $any_opt=0; foreach my $arg(@ARGV) { if($arg=~/^VER=(\d+)$/i) { $cut_off_ver=$1; $any_opt=1; } if($arg=~/^SOURCE_LOC=(.+?)(\/)?$/i) { $source_dir_path=$1; $any_opt=1; } if($arg=~/^MirrorUp_LOC=(.+?)(\/)?$/i) { $back=$1; $back=~s/\/MirrorUp\/$|\/MirrorUp$|\/$//gi; $bak_dir=$back; $MirrorUp_dir_location=$back."/MirrorUp"; $any_opt=1; } if($arg=~/^CHECK_DATE=(.+)$/i) { $current_date_check=lc($1); $any_opt=1; } if($arg=~/^STABLE_CUTOFF=(\d+)$/i) { $stable_cutoff=$1; $any_opt=1; } if($arg=~/^-h$/i) { $help=1; $any_opt=1; } if($arg=~/^-v$/i) { $c_ver=1; $any_opt=1; } if($arg=~/^i=(.+)$/i) { $interactive=$1; if($interactive =~/1|yes/i) { $interactive=1; $ask="YES"; } else { $interactive =0; $ask="NO"; } $any_opt=1; } } my $do; my $opts_arg=scalar(@ARGV); my $args=join(" ",@ARGV); if($any_opt==0 && $opts_arg > 0) { print "InValid Option(s) Provided : $args\n"; &USAGE(); } if($current_date_check eq '0' || $current_date_check eq "no") { $do="NO"; $current_date_check=0; } else { $do="YES"; $current_date_check=1; } if($help) { &USAGE(); } if($c_ver) { print STDERR "\n$0 Version $VERSION\n\n"; exit(0); } print STDERR "\nRunning MirrorUp Wizard...\nCreates Mirror of Your Latest Files and Directories.\n** Version v$VERSION **\n\n"; print STDERR "SettingUp MirrorUp Configuration Variables...\n\n"; print STDERR "=================================================================\n"; print STDERR "Description of Config -> CONFIG_ARGUMENT_NAME=Value\n"; print STDERR "----------------------------------------------------\n"; print STDERR "Interactive Mode -> I=$ask\n"; print STDERR "Source Location -> SOURCE_LOC=$source_dir_path\n"; print STDERR "MirrorUp Location -> MIRRORUP_LOC=$MirrorUp_dir_location\n"; print STDERR "Versions to Keep -> VER=$cut_off_ver\n"; print STDERR "MirrorUp Date Check -> CHECK_DATE=$do\n"; print STDERR "File/Dir Stable Cutoff -> STABLE_CUTOFF=$stable_cutoff\n"; print STDERR "=================================================================\n\n"; unless(-d $source_dir_path && -r $source_dir_path) { print STDERR "ERROR: SOURCE_LOCATION (SOURCE_LOC=$source_dir_path) [Not a Directory Or Do Not have Read Permission!]\n\n"; exit(111); } if(defined $back) { unless(-d $back) { print STDERR "ERROR: MirrorUp_LOCATION (MIRRORUP_LOC=$back) [$!]\n\n"; exit(112); } } unless(-r $bak_dir && -w $bak_dir) { print STDERR "ERROR: MIRROR LOCATION ($bak_dir) [Do Not have Read/Write Permission!]\n\n"; exit(113); } unless(defined $back) { $back=$MirrorUp_dir_location; } if($interactive) { print STDERR "Press (y/n) to Take MirrorUp : "; my $inp=&trim(lc()); if($inp ne "y") { print "\nExiting $0...\n\n"; exit 0; } } print "\n###################################################################\n"; print " Date of MirrorUp: $date \n"; print "###################################################################\n\n"; ##################################################################### ############# DO NOT CHANGE ANYTHING BELOW THIS LINE ################ ##################################################################### my $date_file=`date +%Y_%m_%d-%H:%M:%S`; chomp $date_file; my $curr_date=`date +%Y%m%d`; chomp $curr_date; my $cut_off_date=&calc_days($curr_date); #print "IICUT:$cut_off_date\n"; $cut_off_date=$cut_off_date - $stable_cutoff; #print "CUT:$cut_off_date\n"; ################# MirrorUp Directory Structure ######################## #MirrorUp directory location (Location where the MirrorUp directories will be created). $MirrorUp_dir_location=&trim($MirrorUp_dir_location); my $MirrorUp_dir_location_status=&create_dir($MirrorUp_dir_location); if($MirrorUp_dir_location_status !~ /EXIST|NEW/) { exit(11); } my $MirrorUp_dir_name=&generate_MirrorUp_dir_name($source_dir_path); my $MirrorUp_dir_stable=$MirrorUp_dir_location."/MirrorUp_".$MirrorUp_dir_name."_stable"; my $MirrorUp_dir_stable_status=&create_dir($MirrorUp_dir_stable); my $MirrorUp_dir_latest=$MirrorUp_dir_location."/MirrorUp_".$MirrorUp_dir_name."_latest"; my $MirrorUp_dir_latest_status=&create_dir($MirrorUp_dir_latest); my $log_dir=$MirrorUp_dir_location."/MirrorUp_logs"; my $log_dir_status=&create_dir($log_dir); if($MirrorUp_dir_stable_status!~ /EXIST|NEW/ || $MirrorUp_dir_latest_status !~ /EXIST|NEW/ || $log_dir_status !~ /EXIST|NEW/) { exit(12); } ################### Checking Today's MirrorUp ######################### if($log_dir_status ne "NEW" || $MirrorUp_dir_location_status ne "NEW") { if($current_date_check) { #To take MirrorUp Once in a day [Resticts multiple MirrorUp in a day] my $MirrorUp_date=$date_file; $MirrorUp_date=~ s/^(.+)\-.+/$1/g; my $res=`ls $log_dir | grep "$MirrorUp_date"`; if($res ne "") { print "\nFound Log File(s) having Today's Date : \n$res\n\n"; print "You have Already Taken the MirrorUp It Seems !!!\n"; sleep 1; print "\nExiting $0...\n\n"; exit(0); } } } ##################################################################### # Opening Log file... my $MirrorUp_log_file=$log_dir."/MirrorUpLog_".$MirrorUp_dir_name."_".$date_file; open(PR_BACK,">$MirrorUp_log_file")|| die "Can't Open $MirrorUp_log_file : $!\n"; print PR_BACK "=================================================================\n"; print PR_BACK "Description of Config -> CONFIG_ARGUMENT_NAME=Value\n"; print PR_BACK "----------------------------------------------------\n"; print PR_BACK "Interactive Mode -> I=$ask\n"; print PR_BACK "Source Location -> SOURCE_LOC=$source_dir_path\n"; print PR_BACK "MirrorUp Location -> MIRRORUP_LOC=$MirrorUp_dir_location\n"; print PR_BACK "Versions to Keep -> VER=$cut_off_ver\n"; print PR_BACK "MirrorUp Date Check -> CHECK_DATE=$do\n"; print PR_BACK "File/Dir Stable Cutoff -> STABLE_CUTOFF=$stable_cutoff\n"; print PR_BACK "=================================================================\n\n"; ######################## SIZE OF SOURCE LOC ######################## #du -sk . | awk '{ $res=$1; if ( $res >= 1024 ){ $res= $res/1024; print $res , " MB ", $2 } else { print $res, " KB " , $2 } }' &get_dir_size("Source",$source_dir_path); #################### Source : MirrorUp Directory ###################### #Source Directory to be backed up. # Default is PWD of MirrorUp script IF $source_dir_path is NOT defined # Gets the source directory listings. #drwxr-xr-x 2 ssahu asp 96 2008-07-23 08:32:36 /home/ssahu/dev_san/BOX my $MirrorUp_dir=&get_dir_listings($source_dir_path); # All documents of current directory... my @MirrorUp_dir=split(/\n/,$MirrorUp_dir); my %MirrorUp_files_stable; #docs 1 month old my %MirrorUp_files_latest; #docs current month, recently modified. my %parent_dir_flag; #Acknowledge the directory type (Which parent directory/file already copied) ######################### SORTING OUT DOCS ########################## # Sorting the Documents accordingly to Latest AND Stable type. foreach my $dir_file (sort @MirrorUp_dir) { $dir_file=&trim($dir_file); #print "====$dir_file---\n"; # filtering only dir and scripts not links. if($dir_file=~ /^d|^-/) { # parsing file attributes. my ($file_size_byte,$date,$time_stamp)=&list_parse($dir_file); $date=~ s/-//g; my $name=&get_filename($dir_file); #print "====$name----\n"; next unless(defined $name); my $cut_date=&calc_days($date); # sorting old and latest docs. if($cut_date <= $cut_off_date) { # 1 month old/ stable files. $MirrorUp_files_stable{$dir_file}=&trim($name); #print "STABLE: $dir_file: $name\n"; } else { $MirrorUp_files_latest{$dir_file}=&trim($name); #print "latest files :$MirrorUp_files_latest{$dir_file}\n"; } } } ############################# MirrorUp PROCESS ####################### # if The MirrorUp runs for the first time or any of one dir (stable or latest )is new one. if($MirrorUp_dir_stable_status eq "NEW" || $MirrorUp_dir_latest_status eq "NEW") { print "\n"; foreach my $file (sort keys %MirrorUp_files_latest) { my $destination_doc=&get_parent_dir($file, $source_dir_path, $MirrorUp_dir_latest ,"1"); my $source_parent_doc=&get_parent_dir($file, $source_dir_path, $MirrorUp_dir_latest ,"0"); my $res; my $skip=&check_MirrorUpdir($source_parent_doc); if ($skip eq "YES") { #print "Skipping MirrorUp : $source_parent_doc\n"; $parent_dir_flag{$source_parent_doc}=1; next; } if(!defined $parent_dir_flag{$source_parent_doc} || $parent_dir_flag{$source_parent_doc} ne "1") { my $pr_str="$source_parent_doc to $destination_doc"; my $file_print=&get_sized_string($pr_str,100); print "Latest MirrorUp : $file_print\n"; #print "Latest MirrorUp : $source_parent_doc to $destination_doc\n"; $res=&execute_command("cp", "-pr", "$source_parent_doc $destination_doc"); $parent_dir_flag{$source_parent_doc}=1; } else { $res=0; } chomp $res; if($res eq '0') { $MirrorUp_files_latest{$file}="Done"; } else { $MirrorUp_files_latest{$file}="Error"; } } foreach my $file (sort keys %MirrorUp_files_stable) { my $destination_doc=&get_parent_dir($file, $source_dir_path, $MirrorUp_dir_stable, "1"); my $source_parent_doc=&get_parent_dir($file,$source_dir_path, $MirrorUp_dir_stable ,"0"); my $res; my $skip=&check_MirrorUpdir($source_parent_doc); if ($skip eq "YES") { print "Skipping MirrorUp : $source_parent_doc\n"; $parent_dir_flag{$source_parent_doc}=1; next; } if(!defined $parent_dir_flag{$source_parent_doc} || $parent_dir_flag{$source_parent_doc} ne "1") { my $pr_str="$source_parent_doc to $destination_doc"; my $file_print=&get_sized_string($pr_str,100); print "Stable MirrorUp : $file_print\n"; #print "Stable MirrorUp : $source_parent_doc to $destination_doc\n"; $res=&execute_command("cp", "-pr", "$source_parent_doc $destination_doc"); $parent_dir_flag{$source_parent_doc}=1; } else { $res=0; } chomp $res; if($res eq '0') { $MirrorUp_files_stable{$file}="Done"; } else { $MirrorUp_files_stable{$file}="Error"; } } print PR_BACK "*******************************************************************\n"; print PR_BACK " The Latest Documents (Lessthan $stable_cutoff days Old) \n"; print PR_BACK "*******************************************************************\n\n"; my $ldr=&print_log(\%MirrorUp_files_latest, $MirrorUp_dir_latest); #print PR_BACK "None.\n" if (!$ldr); print PR_BACK "*******************************************************************\n"; print PR_BACK " The Stable Documents (Morethan $stable_cutoff days Old) \n"; print PR_BACK "*******************************************************************\n\n"; my $lss=&print_log(\%MirrorUp_files_stable,$MirrorUp_dir_stable); #print PR_BACK "None.\n" if (!$lss); } ###################################################################### # When MirrorUp dirs found if($MirrorUp_dir_stable_status eq "EXIST" && $MirrorUp_dir_latest_status eq "EXIST") { print "\n"; my $latest_dir=&get_dir_listings($MirrorUp_dir_latest); my %latest_dir=&get_files_hash($latest_dir); #MirrorUp latest directory listing my $stable_dir=&get_dir_listings($MirrorUp_dir_stable); my %stable_dir=&get_files_hash($stable_dir); #MirrorUp Stable directory listing ##################################################################### # Actively changed documents... my %move_latest_existing; my %latest_to_copy; my %newdocs_to_copy; my %version_MirrorUp_files; my %files_1m_stable_move; my %files_1m_stable_VER_REMOVE; my %files_1m_stable_REMOVE; my %newdocs_flag; my %files_1m_stable_move_temp1; my %files_1m_stable_move_temp2; my %dir_stable_flag; my %dir_latest_flag; my %files_vers_latest_dir; my %REMOVE_version_files; ##################################################################### #Cross checking the original dir files with latest MirrorUp dir files. foreach my $MirrorUp_lsl (keys %MirrorUp_files_latest) { my $mimc_MirrorUp_latest=$MirrorUp_files_latest{$MirrorUp_lsl}; $mimc_MirrorUp_latest=~ s/^$source_dir_path\//$MirrorUp_dir_latest\//; my $doc_test=&get_parent_dir($MirrorUp_lsl ,$bak_dir, "","0"); my $skip=&check_MirrorUpdir($doc_test); if ($skip eq "YES") { delete $MirrorUp_files_latest{$MirrorUp_lsl}; #print "Skipping MirrorUp : $mimc_MirrorUp_latest\n"; next; } if(defined $latest_dir{$mimc_MirrorUp_latest}) { #print "==$MirrorUp_lsl==\n"; next if($MirrorUp_lsl=~ /^d/); my ($file_size_byte1,$date1,$time_stamp1)=&list_parse($latest_dir{$mimc_MirrorUp_latest}); my $latest_timestamp_old=$file_size_byte1.$date1.$time_stamp1; my ($file_size_byte2,$date2,$time_stamp2)=&list_parse($MirrorUp_lsl); my $MirrorUp_files_timestmp_new=$file_size_byte2.$date2.$time_stamp2; if($latest_timestamp_old ne $MirrorUp_files_timestmp_new) { #print "latest_dir:$latest_timestamp_old===dev_dir:$MirrorUp_files_timestmp_new\n"; my ($file_size_byte,$date,$time_stamp)=&list_parse($latest_dir{$mimc_MirrorUp_latest}); my $file_version="#".$file_size_byte."_".$date."_".$time_stamp; my $filename=$mimc_MirrorUp_latest; my $filename_timestamp=$filename.$file_version; #files needs to renamed $move_latest_existing{$mimc_MirrorUp_latest}=$filename_timestamp; #print "move_latest_existing: $mimc_MirrorUp_latest : $filename_timestamp\n"; #files needs to copied from original dir. $latest_to_copy{$MirrorUp_files_latest{$MirrorUp_lsl}}=$mimc_MirrorUp_latest; #print "Latest to copy :$MirrorUp_files_latest{$MirrorUp_lsl}: $mimc_MirrorUp_latest\n"; } } else { my ($source_doc, $destination_doc)=&MirrorUp_newdocs($MirrorUp_files_latest{$MirrorUp_lsl} ,$source_dir_path, $MirrorUp_dir_latest); $newdocs_to_copy{$source_doc}=$destination_doc; #print "NEW--$MirrorUp_files_latest{$MirrorUp_lsl}--\n"; } } #print Dumper(\%MirrorUp_files_latest); ###################################################################### # MirrorUp privious version to it's time stampand current file name as it's original name my $move_latest_val=scalar(keys %move_latest_existing); if($move_latest_val>0) { print "BackingUp Privious Versions Documents With it's TimeStamp\n"; print "===================================================================\n"; print PR_BACK "\nBackingUp Privious Versions Documents With it's TimeStamp\n"; print PR_BACK "===================================================================\n"; foreach my $rename (keys %move_latest_existing) { $process_flg=1; print "BackingUp Privious Versions : \"$rename\" to \"$move_latest_existing{$rename}\"\n"; print PR_BACK "BackingUp Privious Versions : \"$rename\" to \"$move_latest_existing{$rename}\"\n"; my $str="BackingUp : $rename to $move_latest_existing{$rename}"; my $file_print=&get_sized_string($str,100); #print "BackingUp Versions : $file_print\n"; #sleep 1; my $res=&execute_command("mv", "-f", "$rename $move_latest_existing{$rename}"); } } ##################################################################### # Latest Version copy... my $latest_to_copy_val=scalar(keys %latest_to_copy); if($latest_to_copy_val>0) { print "\nLatest Version File(s) MirrorUp from \"$source_dir_path\" to \"$MirrorUp_dir_latest\"\n"; print "===================================================================\n"; print PR_BACK "\nLatest Version File(s) MirrorUp from \"$source_dir_path\" to \"$MirrorUp_dir_latest\"\n"; print PR_BACK "===================================================================\n"; foreach my $new_entry (keys %latest_to_copy) { $process_flg=1; #print "Latest MirrorUp File : \"$new_entry\"\n"; print PR_BACK "Latest Version MirrorUp : \"$new_entry\" to \"$latest_to_copy{$new_entry}\"\n"; my $str="Latest Version MirrorUp : \"$new_entry\" to \"$latest_to_copy{$new_entry}\""; print "$str\n"; my $file_print=&get_sized_string($str,100); #print "Latest MirrorUp : $file_print\n"; my $res=&execute_command("cp", "-pr", "$new_entry $latest_to_copy{$new_entry}"); } } ##################################################################### # Newly added documents copy to MirrorUp latest dirctory foreach my $source_name (sort keys %newdocs_to_copy) { my $res; if(!defined $newdocs_flag{$source_name}) { $process_flg=1; print PR_BACK "Latest New MirrorUp : \"$source_name\" to \"$newdocs_to_copy{$source_name}\"\n"; print "Latest New MirrorUp : \"$source_name\" to \"$newdocs_to_copy{$source_name}\"\n"; $res=&execute_command("cp", "-rp", "$source_name $newdocs_to_copy{$source_name}"); $newdocs_flag{$source_name}=1; } else { $res=0; } if($res eq "0") { $newdocs_flag{$source_name}="Done"; } else { $newdocs_flag{$source_name}="Error"; } } ###################################################################### # for 1 month old stable file in latest dir with consideration of # all files in the parent directory### foreach my $dir_file (sort values %latest_dir) { if($dir_file=~ /^d|^-/) { #print "latest_dir:$dir_file\n"; my ($file_size_byte,$date,$time_stamp)=&list_parse($dir_file); $date=~ s/-//g; my $name=&get_filename($dir_file); next unless(defined $name); # sorting old and latest docs. #4575_2008-07-12_22:22:00== if($name !~ m|\#\d+_\d{4}-\d{2}-\d{2}_\d{2}:\d{2}:\d{2}$|) { #print"==$name==\n"; my $destination_doc=&get_parent_dir($dir_file, $MirrorUp_dir_latest, $MirrorUp_dir_latest, "1"); if($date < $cut_off_date) { # 1 month old/ stable files and ready to move to stable dir. $files_1m_stable_move_temp1{$dir_file}=$name; #print "Name: $name\n"; $dir_stable_flag{$destination_doc}=1; #print "1m_old_stable :$destination_doc\n"; } else { $dir_latest_flag{$destination_doc}=1; } } } } foreach my $move_file (sort keys %files_1m_stable_move_temp1) { my $destination_doc=&get_parent_dir($move_file, $MirrorUp_dir_latest, $MirrorUp_dir_latest, "1"); #print "Move_file:$move_file\n"; unless(defined $dir_latest_flag{$destination_doc}) { my $name=&get_filename($move_file); next unless(defined $name); $files_1m_stable_move_temp2{$name}=$move_file; #print "1 month old Stable at Latest dir: $name\n"; foreach my $name_all (sort keys %latest_dir) { #print "--$name_all--\n"; if($name_all=~ m|(^$name\#\d+_\d{4}-\d{2}-\d{2}_\d{2}:\d{2}:\d{2}$)|) { my $ver_filename=$1; $files_1m_stable_VER_REMOVE{$ver_filename}=1; #print "REMOVE:$ver_filename\n"; } } } } %files_1m_stable_move=&determine_docs_to_move(\%files_1m_stable_move_temp2,$MirrorUp_dir_latest, $MirrorUp_dir_stable); #print Dumper(\%files_1m_stable_move); ###################################################################### # Removing Unwanted old version files from latest dir my $val=scalar(keys %files_1m_stable_VER_REMOVE); if($val>0) { print "\nRemoving UnWanted Old Version files from \"$MirrorUp_dir_latest\"\n"; print "===================================================================\n"; print PR_BACK "\nRemoving UnWanted Old Version files from \"$MirrorUp_dir_latest\"\n"; print PR_BACK "===================================================================\n"; foreach my $remove_file (keys %files_1m_stable_VER_REMOVE) { $process_flg=1; my $file_print=&get_sized_string($remove_file,100); print PR_BACK "Removing : $remove_file\n"; print "Removing : $remove_file\n"; #print "\rRemoving : $file_print\r"; #sleep 1; my $res=&execute_command("rm", "-rf", $remove_file); } } # Moving of stable 1 Month old file to stable dir my $files_1m_stable_move_val=scalar(keys %files_1m_stable_move); if($files_1m_stable_move_val>0) { #print "$files_1m_stable_move_val\n"; print "\nMoving $stable_cutoff days Stable Documnent from \"$MirrorUp_dir_latest\" to \"$MirrorUp_dir_stable\"\n"; print "===================================================================\n"; print PR_BACK "\nMoving $stable_cutoff days Stable Documnent from \"$MirrorUp_dir_latest\" to \"$MirrorUp_dir_stable\"\n"; print PR_BACK "===================================================================\n"; } foreach my $name (keys %files_1m_stable_move) { $process_flg=1; my $path_file_stable=$name; $path_file_stable=~ s/^$MirrorUp_dir_latest\//$MirrorUp_dir_stable\//; #print "===$path_file_stable===\n"; if(defined $stable_dir{$path_file_stable}) { $process_flg=1; print "Removing Old Entry : \"$path_file_stable\".\n"; print PR_BACK "Removing Old Entry : \"$path_file_stable\".\n"; my $file_print=&get_sized_string($path_file_stable,100); #print "Removing : $file_print\n"; #sleep 1; my $res=&execute_command("rm" , "-rf" , $path_file_stable); } print "Moving Stable Entry :\"$name\" to \"$MirrorUp_dir_stable\"\n"; print PR_BACK "Moving Stable Entry : \"name\" to \"$MirrorUp_dir_stable\".\n"; my $file_pr=&get_sized_string($name,100); #print "Moving : $file_pr\n"; #sleep 1; my $res=&execute_command("mv", "-f", "$name $path_file_stable"); } ###################################################################### #cutting off threshold version files of latest MirrorUp file $latest_dir=&get_dir_listings($MirrorUp_dir_latest); %latest_dir=&get_files_hash($latest_dir); #MirrorUp latest directory listing foreach my $file (sort keys %latest_dir) { if($file=~ /^(.+)\#\d+_\d{4}-\d{2}-\d{2}_\d{2}:\d{2}:\d{2}$/) { my $filename=$1; #print "==$filename--::$file\n"; $files_vers_latest_dir{$filename}{$file}=1; } } foreach my $file (sort keys %files_vers_latest_dir) { my $count_ver=scalar(keys %{$files_vers_latest_dir{$file}}); my $size=$cut_off_ver-1; #print "$count_ver > $size \n"; if($count_ver > $size ) { #print "$count_ver : $file\n"; my %ver_timestamp; foreach my $ver (sort keys %{$files_vers_latest_dir{$file}}) { my $timestamp=&get_timestamp($ver); $ver_timestamp{$timestamp}{$ver}=1; #print "$timestamp:$ver\n"; } my $count_val=1; foreach my $time (sort {$b<=>$a} keys %ver_timestamp) { #print "time:$time\n"; if($count_val< $cut_off_ver) { #print "$count_val: NEXT : $time\n"; $count_val++; next; } else { foreach my $vers (keys % { $ver_timestamp{$time} }) { $REMOVE_version_files{$vers}=1; #print "$count_val : REMOVE_VERSION : $time : $vers\n"; $count_val++; } } } } } my $val_rem_ver=scalar(keys %REMOVE_version_files); if($val_rem_ver>0) { print "\nRemoving Old Version files from \"$MirrorUp_dir_latest\"\n"; print "===================================================================\n"; print PR_BACK "\nRemoving Old Version files from \"$MirrorUp_dir_latest\"\n"; print PR_BACK "===================================================================\n"; } foreach my $remove_ver (keys %REMOVE_version_files) { $process_flg=1; my $file_print=&get_sized_string($remove_ver,100); print "Removing Version file : $remove_ver\n"; print PR_BACK "Removing Version file : $remove_ver\n"; #print "\rRemoving Version : $file_print\r"; #sleep 1; my $res=&execute_command("rm", "-rf", "$remove_ver"); } my $MirrorUp_dir_latest_final=&get_dir_listings($MirrorUp_dir_latest); my %MirrorUp_dir_latest_final=&get_files_hash($MirrorUp_dir_latest_final); my $MirrorUp_dir_stable_final=&get_dir_listings($MirrorUp_dir_stable); my %MirrorUp_dir_stable_final=&get_files_hash($MirrorUp_dir_stable_final); ###################################################################### if(!$process_flg) { print "\nNo Latest Or $stable_cutoff days Old Stable Documnets Found.\n"; print PR_BACK "\nNo Latest Or $stable_cutoff days Old Stable Documnets Found.\n"; print "Nothing to MirrorUp : $date.\n\n"; print PR_BACK "Nothing to MirrorUp : $date.\n\n"; } print PR_BACK "\n*******************************************************************\n"; print PR_BACK " The Latest Documents (Lessthan $stable_cutoff days Old) \n"; print PR_BACK "*******************************************************************\n\n"; my $lr=&print_log(\%MirrorUp_dir_latest_final,$MirrorUp_dir_latest,"reverse"); #print PR_BACK "None.\n" if (!$lr); #print PR_BACK "$MirrorUp_dir_latest_final\n\n"; print PR_BACK "\n*******************************************************************\n"; print PR_BACK " The Stable Documents (Morethan $stable_cutoff days Old) \n"; print PR_BACK "*******************************************************************\n\n"; my $l=&print_log(\%MirrorUp_dir_stable_final,$MirrorUp_dir_stable,"reverse"); #print PR_BACK "None.\n" if (!$l); #print PR_BACK "$MirrorUp_dir_stable_final\n\n"; } ###################################################################### print STDERR "\n"; &get_dir_size("MirrorUp", $MirrorUp_dir_location); print STDERR "NOTE : Created MirrorUp log File : $MirrorUp_log_file\n\n"; #system("vi $MirrorUp_log_file"); print STDERR "BackingUp Process Completed...\n\n"; print PR_BACK "BackingUp Process Completed...\n\n"; exit(0); ###__END__### ############################# Create Dir ############################# =head * Creates new directory or find directory if exist. =cut sub create_dir { my ($path_dir_name)=@_; unless(-d $path_dir_name) { my $res_dir=&execute_command("mkdir", "-m 755", $path_dir_name); if(!$res_dir) { print "Creating MirrorUp Directory : \"$path_dir_name\"\n"; #sleep 1; return "NEW"; } else { print STDERR "Error Creating Directory : \"$path_dir_name\" !!!\n\n"; return $res_dir; } } else { print "Found MirrorUp Directory : \"$path_dir_name\"\n"; #sleep 1; return "EXIST"; } } ############################ Print Logs ############################## =head * Distingushes the files and directories from the hash given and writes into log file * "reverse" will reverse the hash input given (keys <-> values) =cut sub print_log { my ($MirrorUp_file_dir,$path,$map)=@_; my %MirrorUp_file_dir=%{$MirrorUp_file_dir}; my %temp; my $any=0; if(defined $map && $map eq "reverse") { foreach my $key (keys %MirrorUp_file_dir) { $temp{$MirrorUp_file_dir{$key}}="Done"; } %MirrorUp_file_dir=%temp; } print PR_BACK "MirrorUp Directories: $path\n"; print PR_BACK "===================================================================\n\n"; my $any_d=0; my $count=1; foreach my $file (sort keys %MirrorUp_file_dir) { my $file_attr=$MirrorUp_file_dir{$file}; if($file=~ /^d/) { $any=1; $any_d=1; my $count_p=sprintf("%04s",$count); print PR_BACK "$count_p : $file_attr $file\n"; $count++; } } print PR_BACK "None.\n" if(!$any_d); print PR_BACK "\n\nMirrorUp Files: $path\n"; print PR_BACK "===================================================================\n\n"; my $any_f=0; foreach my $file (sort keys %MirrorUp_file_dir) { my $file_attr=$MirrorUp_file_dir{$file}; $file=&trim($file); if($file !~ /^d/) { $any_f=1; $any=1; my $count_p=sprintf("%04s",$count); print PR_BACK "$count_p : $file_attr $file\n"; $count++; } } print PR_BACK "None.\n" if(!$any_f); print PR_BACK "\n\n"; return $any; } ######################### List Parser ################################ # Parses the file attributes. #-rw-r--r-- 1 eqpacea prod_grp 2149 2008-07-31 06:53:27 /home/ssahu/dev_san/status_rep/week46.txt.log sub list_parse { my ($file)=@_; my ($dir_and_perm,$links_file,$owner,$group,$file_size_byte,$date,$time_stamp,$file_name)=split(/\s+/,$file,9); #print "dr:$dir_and_perm,ln:$links_file,ow:$owner,gr:$group,si:$file_size_byte,dt:$date,ti:$time_stamp,fi:$file_name\n"; return ($file_size_byte,$date,$time_stamp); } ############################ get_sized_string ####################### =head 1. take input as file name or string 2. If the file length is maintained as 40 (say).[ $cutoff_len=40 ], can be changed. 3. If length is more than 40 the last three caracter is substituted with ... 4. if lessthen 40 the remaining vacant part is filled by white spaces. =cut sub get_sized_string { my ($file,$cutoff_len)=@_; my $len=length($file); #cutoff size (may be changed to desired size.) my $substr_len=$cutoff_len-1; my $file_pr=substr($file,0,$substr_len); #print "$len: $file_pr\n"; if($len>$cutoff_len) { $file_pr=~ s/^(.+)\S{3}$/$1\.\.\./; } elsif($len<$cutoff_len) { my $diff=$cutoff_len-$len; $file_pr=sprintf("%s%$diff.s",$file_pr," "); } return $file_pr; } ###################################################################### #trim space/new line/tab from either ends of string sub trim { my $word = shift; $word =~ s/^\s*|\s*$//g if defined $word && $word ne ""; $word =~ s/^\n*|\n*$//g if defined $word && $word ne ""; return $word; } ###################################################################### #returns the filename with it's absolute path from input lile below: #drwxrwxr-x 2 tricom tricom 4096 2007-03-29 20:26 /path/MirrorUp_current_week sub get_filename { my ($file_attr)=@_; #return undef if(!defined $file_attr || $file_attr !~ /^\S{10}\s+\d+\s+\w+\s+\w+\s+\d+\s+\d{4}\-\d{2}\-\d{2}\s+\d{2}\:\d{2}/); my @arr_for_file_name=split(/\s+/,$file_attr,8); my $filename=pop(@arr_for_file_name); $filename=~ s/ /\\ /g; return $filename; } ###################################################################### #retuns time stamp sub get_timestamp { my ($file)=@_; return undef if (!defined $file); $file=&trim($file); if($file=~ /^(.+)\#\d+_(\d{4}-\d{2}-\d{2}_\d{2}:\d{2}:\d{2})$/) { my $timestamp=$2; $timestamp=~ s/\#|\-|\_|\://g; #print "$timestamp\n"; return ($timestamp); } } ###################################################################### #retuns a hash of files with filename as key sub get_files_hash { my ($MirrorUp_dir_files)=@_; my @MirrorUp_dir=split(/\n/,$MirrorUp_dir_files); my %list_dir; foreach my $file (@MirrorUp_dir) { $file=&trim($file); if($file=~ /^d|^-/) { my $name=&get_filename($file); next unless(defined $name); $list_dir{$name}=$file; } } return %list_dir; } ##################################################################### sub get_year_month_date_time { my ($str_grp, $path)=@_; #print "INPUT--$str_grp===$path--\n"; #drwxr-xr-x 2 ssahu asp 96 Dec 13 2007 MirrorUp #drwxrwxr-x 2 tricom tricom 4096 2007-03-29 20:26 MirrorUp_current_week my %month=( 'Jan' => '01', 'Feb' => '02', 'Mar' => '03', 'Apr' => '04', 'May' => '05', 'Jun' => '06', 'Jul' => '07', 'Aug' => '08', 'Sep' => '09', 'Oct' => '10', 'Nov' => '11', 'Dec' => '12' ); my @ls_initial=split(/\n/,$str_grp); my @ls_final; foreach my $str (@ls_initial) { next if($str=~ /^total\s+\d+$/); my @arr_for_index=split(/\s+/,$str); my $index=scalar(@arr_for_index); my @arr_for_file_name=split(/\s+/,$str,9); my $file_name=pop(@arr_for_file_name); my $rev=reverse($str); my $num_split=($index-9)+5; my @attr=split(/\s+/,$rev,$num_split); my $file_initial_attr=reverse(pop(@attr)); $file_initial_attr=~ s/\d+$//g; $path=&trim($path); $path=~ s/\/$//g; $path=$path."/"; my $file_name_with_path=$path.$file_name; my %attr_hash; my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size,$atime, $mtime,$ctime,$blksize,$blocks)=stat("$file_name_with_path"); my $now = ctime($mtime); #Thu May 8 03:42:15 2008 my ($year, $month, $day, $time); if($now=~ /^\w{3}\s+(\w{3})\s+(\d+)\s+(\d{2}:\d{2}:\d{2})\s+(\d{4})/) { $month=$month{$1}; $day=sprintf("%02d", $2); $time=$3; $year=$4; } my $timestamp=$year."-".$month."-".$day." ".$time; #drwxr-xr-x 2 ssahu asp 96 Dec 13 2007 MirrorUp #print "##$file_initial_attr==$size##$timestamp==$file_name##\n"; my $new_str=$file_initial_attr."$size $timestamp".' '.$file_name_with_path; #print "$new_str\n"; push(@ls_final,$new_str); } my $ls_string=join("\n",@ls_final); return $ls_string; } ##################################################################### #get the file listings in a quniq format like : #drwxr-xr-x 2 ssahu asp 96 2007-03-29 20:26 /home/ssahu/MirrorUp sub get_dir_listings { my ($path)=@_; my @listings=`ls -Rlrt $path`; #my $c=1; my $dir=""; my $flg=0; my @final_listings=(); foreach my $li (@listings) { $li=&trim($li); next if($li=~ /^total\s+\d+$/ || $li eq "" || $li=~/^l/i); if($li=~ /(.+):$/) { $dir=$1; $flg=1; #print "Found Dir: $dir\n"; next; } if($flg==1) { my $file_with_attr_path=&get_year_month_date_time($li, $dir); push(@final_listings,$file_with_attr_path); #print "####$file_with_attr_path####\n\n"; #$c++; } } my $final_listings=join("\n",@final_listings); return $final_listings; } ##################################################################### sub MirrorUp_newdocs { my ($file,$source_loc, $MirrorUp_latest_loc)=@_; my $mimic_latest_dir_doc=$file; $mimic_latest_dir_doc=~ s/^$source_loc\//$MirrorUp_latest_loc\//; my @dir_lev=split(/\//,$mimic_latest_dir_doc); my $back_dir_list=""; my $source_dir_list=""; my $count=0; foreach my $dir (@dir_lev) { if($count ne "1") { $back_dir_list=$back_dir_list."/".$dir; } else { $back_dir_list=$back_dir_list.$dir; } if(-e $back_dir_list) { $count++; next; } else { last; } } $source_dir_list=$back_dir_list; $source_dir_list=~ s/^$MirrorUp_latest_loc\//$source_loc\//; return ($source_dir_list, $back_dir_list); } ##################################################################### sub get_parent_dir { my ($file_lsl, $source_path, $MirrorUp_dir, $back_flg)=@_; my $name=&get_filename($file_lsl); #print "INPUT--$name--\n"; $name=~ s/^$source_path//g; my @dir=split(/\//,$name); my $parent_dir=shift(@dir); my $child=shift(@dir); #print "Parent-$parent_dir-Child--$child==\n"; $parent_dir=$parent_dir."/".$child; if($back_flg eq "1") { $parent_dir=$MirrorUp_dir.$parent_dir; #print "1BAC==$parent_dir==\n"; return $parent_dir; } else { $parent_dir=$source_dir_path.$parent_dir; #print "0SOU==$parent_dir==\n"; return $parent_dir; } } ##################################################################### sub determine_docs_to_move { my ($list_hash, $bak_latest,$bak_stable)=@_; my %dir_level_doc; my %final_doc_list=%{$list_hash}; foreach my $file (sort keys %final_doc_list) { my @index=split(/\//,$file); my $level=scalar(@index); $dir_level_doc{$level}{$file}=1; } foreach my $lev (sort {$a<=>$b} keys %dir_level_doc) { foreach my $doc (sort keys %{$dir_level_doc{$lev}}) { foreach my $comp (sort keys %final_doc_list) { if($doc ne $comp) { if($comp=~ /$doc/) { if(-d $doc) { delete $final_doc_list{$comp}; } } } } } } return %final_doc_list; } ##################################################################### #Executes the unix/linux command. #if Debug : $debug=1 then only prints the commands sub execute_command { my ($com , $opt , $arg)=@_; my $res; if($debug) { print "DEBUG (Virtual Command) : $com $opt $arg\n"; $res=0; } else { $res=system("$com $opt $arg"); chomp $res; } if($res eq "0") { return 0; } else { if($res >= 256) { $res=$res/256; return($res); } } } ##################################################################### #Generates Mirror sub-directory name sub generate_MirrorUp_dir_name { my ($MirrorUp_loc)=@_; my @dirs=split(/\//, $MirrorUp_loc); my $name=""; my $num=scalar(@dirs); if($num==0) { $name="su_root"; } else { shift @dirs; my $back_dir=pop(@dirs); foreach my $na (@dirs) { #print "###$na###\n"; my $sub_name=substr($na, 0, 4); $name .=$sub_name."_"; } $name .=$back_dir; } #print "MirrorUp_DIR_NAME===$name===\n"; return $name; } ##################################################################### #returns the size of the file/dir sub get_dir_size { my ($type, $source_dir_path)=@_; my $source_dir_size=`du -sk $source_dir_path`; chomp $source_dir_size; #print "--$source_dir_size--\n"; my ($size, $dir)=split(/\s+/,$source_dir_size); if ( $size >= 1024 ) { my $size_mb =$size/1024; if($size_mb >= 1024 ) { my $size_gb =$size_mb/1024; $size_gb=sprintf("%.2f",$size_gb); print "\n$type Directory Size : $size_gb GB $dir\n\n"; print PR_BACK "\n$type Directory Size : $size_gb GB $dir\n\n"; } else { $size_mb=sprintf("%.2f",$size_mb); print "\n$type Directory Size : $size_mb MB $dir\n\n"; print PR_BACK "\n$type Directory Size : $size_mb MB $dir\n\n"; } } else { $size=sprintf("%.2f",$size); print "\n$type Directory Size : $size KB $dir\n\n"; print PR_BACK "\n$type Directory Size : $size KB $dir\n\n"; } } ##################################################################### #Checks given directory is the one where the mirroring takes place. sub check_MirrorUpdir { my ($doc)=@_; if(-d $doc) { if($MirrorUp_dir_location eq $doc) { return "YES"; } } else { return "NO"; } } ##################################################################### #returns the number of days from YYYYMMDD sub calc_days { my ($date) = @_; my @offset_days = qw(0 31 59 90 120 151 181 212 243 273 304 334); my ($year, $month, $day)= $date=~ m/^(\d{4})(\d{2})(\d{2})$/; #print "$year, $month, $day\n"; my $total_days = $offset_days[$month - 1] + $day + 365 * $year; return $total_days; } sub USAGE { my $var=<