package Storage::FileStorage;

use strict;
use bigint;

use Storage::Storage;
use Logging;
use AgentConfig;
use HelpFuncs;

use POSIX;
use IPC::Run;
use Symbol;

use Storage::Splitter;
use Storage::Counter;

use vars qw|@ISA|;

@ISA = qw|Storage::Storage|;

sub _init {
  my ($self, %options) = @_;

  $self->SUPER::_init(%options);
  $self->{split_size} = $options{split_size};
  $self->{gzip_bundle} = $options{gzip_bundle};
  $self->{output_dir} = $options{output_dir};
  $self->{space_reserved} = $options{space_reserved} if $options{space_reserved};
  $self->{sign} = 1 if $options{sign};

  $self->{last_used_id} = 0;

  $self->{unpacked_size} = 0;
  $self->{packed_size} = 0;

  Logging::info("-" x 60);
  Logging::info("FILE storage initialized.");
  Logging::info("Base directory: $self->{output_dir}");
  Logging::info("Space reserved: $self->{space_reserved}");
  Logging::info("Gzip bundles: " . ($self->{gzip_bundle} ? "yes" : "no"));
  Logging::info("Bundle split size: " . ($self->{split_size} || "do not split"));
  Logging::info("-" x 60);
  $self->reserveSpace();
}

sub getFileSize
{
  my( $fileName ) = @_;
  my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size,$atime,$mtime,$ctime, $blksize,$blocks) = stat($fileName);
  return $size if defined $size;
  return 0;
}

sub reserveSpace {
  my ($self ) = @_;
  if (exists $self->{space_reserved} ) {
    my $avail = (HelpFuncs::getMountSpace($self->{output_dir}))[1];
    if( $avail < $self->{space_reserved} ) {
      my $errmsg = "Available disk space ($avail) is less than required by storage bundle ($self->{space_reserved})";
      Logging::error($errmsg,'fatal');
      die $errmsg;
    }
    my $namebase = $self->{output_dir}.'/.fs_'.(0+$self).'_';
    my $var = 0;
    while( -e "$namebase$var.tmp"){$var++;}
    $self->{space_reserver} = "$namebase$var";
    Logging::info("Reserve disk space at $self->{space_reserver}");
    qx( dd if=/dev/zero of=$self->{space_reserver} bs=$self->{space_reserved} count=1); 
  }
}

sub unreserveSpace {
  my ($self ) = @_;
  if (exists $self->{space_reserver} ) {
    Logging::info("Free reserved disk space at $self->{space_reserver}");
    if( -f $self->{space_reserver}){
      unlink $self->{space_reserver} or Logging::debug("Cannot delete file ".$self->{space_reserver} );
    }
    delete $self->{space_reserver};
  }
  
}

  
#
# Checks the validity of proposed id: it should not be too long.
#

sub getFileNameIdFromId {
  my ($self, $id, $ext, $cansplit ) = @_;

  my $maxLength = &POSIX::PATH_MAX;
  $maxLength -= length( $ext );
  if ( $cansplit && $self->{split_size}) {
    $maxLength -= 4;
  }

  my $destFile = $self->getFullOutputPath() . "/" . $id;
  if (length($destFile) > $maxLength) {
    $id = $self->{last_used_id}++;
    $destFile = $self->getFullOutputPath() . "/" . $id;
  }

  if ($self->{gzip_bundle}) {
    $id .= ":gzipped";
  }

  my $dstDir = $destFile;
  if( $dstDir=~ m/(.*)\/(.*)/ ){
    $dstDir = $1;
    $destFile = $2;
  }
  else{
    $destFile = 'empty';
  }

  return ($dstDir,$destFile, $id);
}

sub getBundleExecutor {
  my ($bundle) = @_;
  return sub {
    my ($exec, $exit_code) = $bundle->run();
    binmode STDOUT;
    my $block;
    my $blocklen;
    my $timeWorking = time();

    while ($blocklen = sysread($exec, $block, 65536)) {
       my $offset = 0;
       do {
         my $written = syswrite(STDOUT, $block, $blocklen, $offset);
         die $! unless defined $written ;
         $offset += $written;
         $blocklen -= $written;
       } while ($blocklen != 0);
       # bug 30101. Prevent ssh connection close(when source host has strong security policy) on big content       
       if ( time() - $timeWorking > 30) {
         Logging::info("Tar is working");
         $timeWorking = time();
       }

    }

    $bundle->cleanup();

    POSIX::_exit($exit_code);
  };
}

sub executeAndSave {
  my ($self, $destDir, $destFile, $destExt, $bundle, $outunpackedSize, $doNotSplit, $doNotGzip ) = @_;

  my $unpackedSize = 0;
  system( "mkdir", "-p", $destDir ) if $destDir and not -e $destDir;

  #allocating filehandle for creating pipe from subprocess
  my $newhandle = POSIX::open("/dev/null", O_RDWR, 0666);

  my @cmd;
  push @cmd, getBundleExecutor($bundle);

  if ($self->{gzip_bundle} and not $doNotGzip ) {
    #FIXME check gzip
    push @cmd, "|", ["gzip"];
  }

  my $files;

  my $newhandle2 = POSIX::open("/dev/null", O_RDWR, 0666);

  my $splitSize = $self->{split_size};
  $splitSize = 0 if $doNotSplit;

  push @cmd, "|", \&Storage::Splitter::run, "$newhandle2>", \$files,
    init => sub {Storage::Splitter::init_process($newhandle2, $splitSize, $destFile, $destDir, $destExt )};

  my $h = IPC::Run::harness(@cmd);

  if (!$h->run()) {
    my ($total, $avail, $mount) = HelpFuncs::getMountSpace($destDir);
    Logging::debug("Failed to pack files $destFile in $destDir [ $avail bytes free of $total bytes total on mount point $mount]");
    Logging::error("Failed to pack files $destFile in $destDir [ $avail bytes free of $total bytes total on mount point $mount]");
    POSIX::close($newhandle);
    POSIX::close($newhandle2);
    return;
  }

  POSIX::close($newhandle);
  POSIX::close($newhandle2);

  if ($unpackedSize =~ /ERR\s(.*)/) {
    Logging::error("Unable to pipe data through filter: $1");
    return;
  }

  $self->{unpacked_size} += $unpackedSize;
  ${$outunpackedSize} = $unpackedSize;

   my @ret;
   foreach my $line ( split/\n/, $files ) {
     my ($file_name, $file_size) = split / /, $line;
      $self->{packed_size} += $file_size;
      my @filedata;
      push @filedata, $file_name;
      push @filedata, $file_size;
      push @ret, \@filedata;
   }
  return \@ret if (@ret);
  return;
}


sub getFullOutputPath{
 my $self = shift;
 return "$self->{output_dir}";
}

sub getFilesFromId{
  my ($self, $id) = @_;
  return $self->{files}->{$id};
}


sub getFilePathFromId{
  my ($self, $id) = @_;
  return $self->{destdir}->{$id};
}

sub getFilesUnpackSizeFromId{
  my ($self, $id) = @_;
  return $self->{unpacksize}->{$id};
}

sub regIdFiles{
  my ($self, $id, $destDir, $unpackedSize, $files, $shortid ) = @_;
  if( $files ){
   $destDir = substr( $destDir, length( $self->getFullOutputPath() ) + 1 ) if index( $destDir, $self->getFullOutputPath() )==0;
   if( index( $destDir, -1, 1 ) eq '/' ) { $destDir = substr( $destDir, 0, length($destDir)-1 ); }
   $id = "$destDir/$id" if $destDir and not $shortid;
   $self->{unpacksize}->{$id} = $unpackedSize;
   $self->{destdir}->{$id} = "$destDir";
   $self->{files}->{$id} = $files;
   for my $file( @{$files} ){
      chmod S_IRUSR|S_IWUSR|S_IRGRP, $self->getFullOutputPath() . '/' . "$destDir/$file->[0]";
   }
   return $id;
  }
  return undef;
}


sub getDumpFiles{
  my ($self, $fromPath ) = @_;

  my @ret;
  while( my( $id, $data ) = each( %{$self->{files}} ) ) {
    my $path = $self->getFilePathFromId( $id );
    $path = substr( $path, length ($fromPath) ) if $fromPath && index( $path, $fromPath )==0;
    $path .= '/' if $path and substr( $path, -1, 1 ) ne '/';
    $path = substr( $path, 1 ) if substr ( $path, 0, 1 ) eq '/';
    foreach my $filedata( @{$data} ) {
    	push @ret, "$path$filedata->[0]";
    }
  }
  return @ret;
}

sub CleanupFiles()
{
  my $self = shift;
  my $pid;
  while( ( $pid = wait() ) !=-1 ){
    Logging::debug("The child process '$pid' has been terminated" );
  }
  my $path = $self->getFullOutputPath();
  my @files = $self->getDumpFiles();
  foreach my $file(@files ){
     Logging::debug("Remove file '$file' from repository '$path' ");
     unlink "$path/$file" or Logging::debug("Cannot remove file '$path/$file'");
  }
  if( exists $self->{discovered} ){
    foreach my $discovered(@{$self->{discovered}} ){
       Logging::debug("Remove discovered '$discovered'");
       opendir DIR, $discovered;
       my @dirfiles = readdir( DIR );
       closedir DIR;
       foreach my $file(@dirfiles){
         if( $file ne '.' and $file ne '..' ){
           unlink "$discovered/$file" or Logging::debug("Cannot remove file '$discovered/$file'");
         }
       }
       rmdir( $discovered ) or Logging::debug("Cannot remove discovered '$discovered'");
    }
  }
}

sub createRepositoryIndex{
  my ( $self, $index ) = @_;
  if( $index ){
    Logging::debug("Create repository index: $index");
    my $destDir = "$self->{output_dir}/.discovered";
    system("mkdir", "-p", "$destDir") if not -e $destDir;
    open INDEXFILE, "> $destDir/$index";
    close INDEXFILE;
  }
}

sub writeDiscovered{
  my ( $self, $dumpPath, $dumpXmlName, $dumpSize, $ownerGuid, $ownerType, $objectGuid, $objectId ) = @_;

  my $idx = rindex( $dumpXmlName, '.xml' );
  $dumpXmlName = substr( $dumpXmlName, 0, $idx ) if $idx>0;
  my $destDir = $self->getFullOutputPath();
  $destDir .= "/$dumpPath" if $dumpPath;
  $destDir .= "/.discovered/$dumpXmlName";
  push @{$self->{discovered}}, $destDir;

  Logging::debug("Create discovered: $destDir");
  system("mkdir", "-p", "$destDir") if not -e $destDir;
  open SIZEFILE, "> $destDir/size_$dumpSize";
  close SIZEFILE;
  open OWNERFILE, "> $destDir/owner_$ownerGuid";
  close OWNERFILE;
  open OWNERTYPEFILE, "> $destDir/ownertype_$ownerType";
  close OWNERTYPEFILE;
  open GUIDFILE, "> $destDir/GUID_$objectGuid";
  close GUIDFILE;
  open OBJIDFILE, "> $destDir/objectid_$objectId";
  close OBJIDFILE;


  my @files;
  my $file = ["size_$dumpSize", 0]; push @files, $file;
  $file = ["owner_$ownerGuid", 0]; push @files, $file;
  $file = ["ownertype_$ownerType", 0]; push @files, $file;
  $file = ["GUID_$objectGuid", 0]; push @files, $file;
  $file = ["objectid_$objectId", 0]; push @files, $file;
  $self->regIdFiles( $destDir, $destDir, 0, \@files );
}


sub getMainDumpXmlFile{
  my ($self) = @_;
  return $self->{dumpxmlfile};
}

sub getDefExtension{
  my ($self) = @_;
  return '' if $self->noDefExtension();
  return '.tgz' if  $self->{gzip_bundle};
  return '.tar';
}

sub addDb {
  my ($self, $proposedId, %options) = @_;

  if ($self->{collectStatistics})
  {
    $self->{stopWatch}->createMarker("pack");
  }

  my ($destDir, $destFile, $id) = $self->getFileNameIdFromId($proposedId, $self->{gzip_bundle}, '', 1);
  Logging::debug("DB bundle. id=$id, destFile=$destFile");

  my $bundle = Storage::Bundle::createDbBundle(%options, 'gzip' => 0 );
  return unless $bundle;
  my $size = 0;
  my $files = $self->executeAndSave($destDir, $destFile, '',  $bundle, \$size, 1, 1 );
  if ($self->{collectStatistics})
  {
    $self->{statistics}->{packTime} += $self->{stopWatch}->getDiff("pack");
    $self->{stopWatch}->releaseMarker("pack");
  }
  if( $files and @{$files} ){
     my $filename = $files->[0]->[0];
     $filename = substr( $filename, length($destDir)+1 ) if index( $filename, $destDir )==0;
     my $ret = $self->addTar( $proposedId, "directory" => $destDir, "include"   => [$filename] );
     foreach my $file( @{$files} ){
       $filename = $file->[0];
       $filename = substr( $filename, length($destDir)+1 ) if index( $filename, $destDir )==0;
       unlink "$destDir/$filename" or Logging::error("Cannot delete temp file '$destDir/$filename'");;
     }
     return $ret;
  }
  else{
    Logging::error("Failed to execute backup of " . $options{'type'} . " database '" . $options{'name'} . "'");
    return undef;
  }

}

sub addTar {
  my ($self, $proposedId, %options) = @_;

  return unless -d $options{'directory'};

  if (defined $options{'checkEmptyDir'}) {
    return unless checkDirForArchive($options{'directory'}, $options{'exclude'}, $options{'include_hidden_files'});
    
  }

  if ($self->{collectStatistics})
  {
    $self->{stopWatch}->createMarker("pack");
  }

  my ($destDir, $destFile, $id) = $self->getFileNameIdFromId( $proposedId, $self->getDefExtension(), 1 );
  Logging::debug("Tar bundle. id=$id, destFile=$destDir/$destFile");

  my $bundle = Storage::Bundle::createTarBundle(%options, 'gzip' => $self->{gzip_bundle});
  return unless $bundle;
  unless ($bundle)
  {
    if ($self->{collectStatistics})
    {
      $self->{statistics}->{packTime} += $self->{stopWatch}->getDiff("pack");
      $self->{stopWatch}->releaseMarker("pack");
    }
    return;
  }
  my $size = 0;
  my $files = $self->executeAndSave($destDir, $destFile, $self->getDefExtension(), $bundle, \$size);
  my $ret =  $self->regIdFiles( $id, $destDir, $size, $files );
  if ($self->{collectStatistics})
  {
    $self->{statistics}->{packTime} += $self->{stopWatch}->getDiff("pack");
    $self->{stopWatch}->releaseMarker("pack");
  }

  return $ret;
}


sub _getInfoXmlFileName {
  my ($self, $fileName ) = @_;
  return "$fileName.xml";
  #return Storage::Splitter::generateUniqueFileName( $fileName, ".xml" );
}

sub finishXmlFile {
  my ($self, $descriptor, $child, $savePath, $fileName) = @_;
  if ($self->{collectStatistics})
  {
    $self->{stopWatch}->createMarker("pack");
  }

  $fileName = 'dump' if not $fileName;
  $savePath = $self->getFullOutputPath() . "/$savePath";
  system("mkdir", "-p", "$savePath") if not -e $savePath;
  my $dumpFile = $self->_getInfoXmlFileName( $fileName );
  my $written;
  my $signBundle = AgentConfig::backupSignUtil() if exists $self->{sign};
  if( $signBundle ){
    if( $child ){ Logging::info("Writing signed child dump file: $savePath/$dumpFile"); }
    else{ Logging::info("Writing signed dump file: $savePath/$dumpFile "); }
    my $handle = Symbol::gensym();
    Logging::debug( "Execute sign bundle: $signBundle" );
    my $pid = open $handle, "|". "$signBundle sign>$savePath/$dumpFile";
    if( $pid and $handle )  {
        if( $child ){  $descriptor->serializeChild($handle, $child); }
        else{ $descriptor->serialize($handle); }
        close($handle);
        waitpid( $pid, 0 );
        if( -s "$savePath/$dumpFile" ){ $written = 1; }
        else{ Logging::error("Cannot sign output file."); }
    }
    else{
       Logging::error("Cannot execute dump file signing.");
    }
  }
  if( not defined $written ){
     Logging::info("Writing dump file: $savePath/$dumpFile");

     open DUMPFILE, "> $savePath/$dumpFile";
     if( $child ){  $descriptor->serializeChild(\*DUMPFILE, $child); }
     else{ $descriptor->serialize(\*DUMPFILE); }
     close DUMPFILE;
  }
  chmod S_IRUSR|S_IWUSR|S_IRGRP, "$savePath/$dumpFile" or Logging::warning("Cannot chmod of '$savePath/$dumpFile'");

  my @files;
  my @file;
  push @file, $dumpFile;
  push @file, getFileSize( "$savePath/$dumpFile" );
  push @files, \@file;
  my $ret = $self->regIdFiles( $dumpFile, $savePath, 0, \@files, $child ? undef : 1 );
  $self->{dumpxmlfile} = $dumpFile if not $child;

  if ($self->{collectStatistics})
  {
    $self->{statistics}->{packTime} += $self->{stopWatch}->getDiff("pack");
    $self->{stopWatch}->releaseMarker("pack");
  }

  return $ret;
}

sub finishChild {
  my $self  = shift;
  my $ret = $self->finishXmlFile( @_ );
  return $ret;
}


sub finish {
  my $self = shift;
  my $descriptor = shift;
  $self->unreserveSpace();
  my $ret = $self->finishXmlFile( $descriptor, undef, @_ );
  return 0 if $ret;
  return 1;
}


sub createContentList{
  my ($self) = @_;
  open CONTENT_FILE, ">" . $self->_getContentListFileName();
  my @files = $self->getDumpFiles( $self->{output_dir} );
  my $fromPath = $self->{output_dir};

  my $fullsize = 0;
  while( my( $id, $data ) = each( %{$self->{files}} ) ) {
    foreach my $filedata( @{$data} ) {
        $fullsize += $filedata->[1];
    }
  }

  print CONTENT_FILE "<contentlist size='$fullsize' >\n";

  while( my( $id, $data ) = each( %{$self->{files}} ) ) {
    my $path = $self->getFilePathFromId( $id );
    $path = substr( $path, length ($fromPath) ) if $fromPath && index( $path, $fromPath )==0;
    $path .= '/' if $path and substr( $path, -1, 1 ) ne '/';
    $path = substr( $path, 1 ) if substr ( $path, 0, 1 ) eq '/';
    foreach my $filedata( @{$data} ) {
        print CONTENT_FILE "  <file size='$filedata->[1]'>$path$filedata->[0]</file>\n";
    }
  }
  print CONTENT_FILE "  <file size='0'>migration.result</file>\n";
  print CONTENT_FILE "</contentlist>\n";
  close CONTENT_FILE;

}


sub getContentList {
  my ($self) = @_;
  if ($self->{collectStatistics})
  {
    $self->{stopWatch}->createMarker("pack");
  }

  open CONTENT_FILE, $self->_getContentListFileName();
  my $s =  join "", <CONTENT_FILE>;
  close CONTENT_FILE;
  if ($self->{collectStatistics})
  {
    $self->{statistics}->{packTime} += $self->{stopWatch}->getDiff("pack");
    $self->{stopWatch}->releaseMarker("pack");
  }
  return $s;
}

sub _getContentListFileName {
  my ($self) = @_;
  return $self->{output_dir} . "/content-list.xml";
}

sub checkDirForArchive {
	my ($srcDir, $exclude, $include_hidden_files) = @_;
	# check that directory is not empty
	if (!opendir(SRCDIR, $srcDir)) {
		return;
	}

	my $filename;
	
	while (defined ($filename = readdir SRCDIR)) {
		my $in_exclude = undef;
		next if $filename =~ /^\.\.?$/;
		if ( ! $include_hidden_files ) {
			next if $filename =~ /^\..*/;
		}
		if ( ref ($exclude) =~ /ARRAY/ ) {
			foreach my $ex (@{$exclude}) {
				$in_exclude = 1 if $filename eq $ex;
			}
			next if defined $in_exclude;
		}
		# directory is not empty
		closedir(SRCDIR);
		return 1;
	}
	# directory is empty
	closedir(SRCDIR);
	return;
}


1;

# Local Variables:
# mode: cperl
# cperl-indent-level: 2
# indent-tabs-mode: nil
# tab-width: 4
# End: