/www/proggenOrg/dedupe/export/trunk/kernel/kernel.cpp

#include "kernel.h"

Dedupe::Core::Kernel::Kernel( Dedupe::FilePath DatabaseLocation,
                                                    size_t Threads,
                                                      std::ostream &Messages )
: Database( DatabaseLocation ),
  Threadcount( Threads ),
  Searcher( ),
  Hasher( ),
  MessageOut( Messages )
{

}

void Dedupe::Core::Kernel::Threadcontroller( std::function<void(Dedupe::FileInfo&)> Func, Dedupe::FileStream &stream )
{
  //if 0 is given, multithreading is disabled
  if( Threadcount == 0u ) std::for_each( stream.begin(), stream.end(), Func );
  else MessageOut << "NOTE:Multithreading not implemented yet" << std::endl;
}

void Dedupe::Core::Kernel::TrackFiles( Dedupe::FilePaths const &Paths, bool recursiv )
{
  boost::function<void ( Dedupe::FileInfo& )>AddFunction;

  AddFunction = boost::bind( &Dedupe::Core::Kernel::AddFile, this, _1 );

  FileAndDirectoryHandlerToFunction( Paths, recursiv, AddFunction );
}

void Dedupe::Core::Kernel::AddFile( Dedupe::FileInfo &Info )
{
  if( Database.AlreadyInDatabase( Info ) )
  {
    MessageOut << "NOTE:" << Info.GetPath() <<": File already in database\n";
    return;
   }

   Hasher.HashFileInfo( Info );

   Database.AddFile( Info );
}

void Dedupe::Core::Kernel::ListDatabase()
{
  Dedupe::FileStream Files( Database.GetFiles());
  for( Dedupe::FileStream::const_iterator it = Files.begin();
                                          it != Files.end();
                                          ++it )
  {
    MessageOut << "LIST:"
               << it->GetPath() << ":"
               << it->GetSize() << ":"
               << it->GetDateChanged() << ":"
               << it->GetType() << ":"
               << it->GetHash() << "\n";
  }

}

Dedupe::FileStream Dedupe::Core::Kernel::GetDatabase()
{
  return Database.GetFiles();
}

void Dedupe::Core::Kernel::UntrackFiles( Dedupe::FilePaths const &Paths, bool recursiv )
{
  boost::function<void ( Dedupe::FileInfo& )>DelFunction;

  DelFunction = boost::bind( &Dedupe::Dataholding::Dataholding::DelFile, &Database, _1 );

  FileAndDirectoryHandlerToFunction( Paths, recursiv, DelFunction );
}


Dedupe::FileStream
Dedupe::Core::Kernel::FileAndDirectoryHandler( Dedupe::FilePaths const &Paths,
                                            bool recursiv)
{
  Dedupe::FileInfo *current;
  Dedupe::FileStream AllFilesFromUserinput;


  for( Dedupe::FilePaths::const_iterator it = Paths.begin(); it != Paths.end(); ++it)
  {
    current = new Dedupe::FileInfo( *it );

    //Catch all FileInfos where the constructor failed( File not existing )
    if( current->GetStatus() != Dedupe::FileInfo::FileOK )
    {
      MessageOut << "ERROR:" << current->GetErrorMessage() << std::endl;
      continue;
    }
    //Push files to the stream
    if( current->GetType() == Dedupe::FileInfo::TFile )
    {
      AllFilesFromUserinput.push_back( *current );
    }
    //if a directory is given, decide between recursive and not
    else if( current->GetType() == Dedupe::FileInfo::TDirectory )
    {
      if( !recursiv ) Searcher.Search( *current );
      else Searcher.SearchRecursive( *current );
      //push founded files to the final Container
      std::for_each(
         Searcher.GetFiles().begin(),
         Searcher.GetFiles().end(),
         [&AllFilesFromUserinput]
         (Dedupe::FileInfo File)
         {AllFilesFromUserinput.push_back( File );});


    }
    delete current;
  }
  return AllFilesFromUserinput;
}


void Dedupe::Core::Kernel::FileAndDirectoryHandlerToFunction( Dedupe::FilePaths const &Paths,
                      bool recursiv,
                      boost::function<void ( Dedupe::FileInfo& )>Function )
{
  Dedupe::FileStream TmpStream = FileAndDirectoryHandler( Paths, recursiv );
  Threadcontroller( Function, TmpStream );
}

Dedupe::Core::Duplicates Dedupe::Core::Kernel::FindHashDuplicates()
{
  //we search for duplicate in the database, decide by hash
  return FindDuplicates( Database.GetFiles(),
                        [](Dedupe::FileInfo lhs, Dedupe::FileInfo rhs )
                            { return lhs.GetHash() > rhs.GetHash(); },
                         [](Dedupe::FileInfo lhs, Dedupe::FileInfo rhs )
                            { return lhs.GetHash() == rhs.GetHash(); });
}

Dedupe::Core::Duplicates Dedupe::Core::Kernel::FindHashDuplicatesExtern(
                                                    Dedupe::FilePaths Paths,
                                                    bool recursiv)
{
  Dedupe::FileStream AllFilesFromUser;

  AllFilesFromUser = FileAndDirectoryHandler( Paths, recursiv );

  std::function<void(Dedupe::FileInfo &Info)>HashFunc =
  std::bind( &Dedupe::Hash::Hash64::HashFileInfo, &Hasher, std::placeholders::_1 );

  Threadcontroller( HashFunc, AllFilesFromUser );

  //we search for duplicate in the given orders, decide by hash
  return FindDuplicates( AllFilesFromUser,
                        [](Dedupe::FileInfo lhs, Dedupe::FileInfo rhs )
                            { return lhs.GetHash() > rhs.GetHash(); },
                         [](Dedupe::FileInfo lhs, Dedupe::FileInfo rhs )
                            { return lhs.GetHash() == rhs.GetHash(); });
}

Dedupe::Core::Duplicates
Dedupe::Core::Kernel::FindFilenameDuplicates()
{
  //we search for duplicates in the database, decide duplicates by filename
  return FindDuplicates( Database.GetFiles(),
              [](Dedupe::FileInfo lhs, Dedupe::FileInfo rhs )
              { return lhs.GetPath().filename() > rhs.GetPath().filename();},

              [](Dedupe::FileInfo lhs, Dedupe::FileInfo rhs )
              { return lhs.GetPath().filename() == rhs.GetPath().filename();});
}


Dedupe::Core::Duplicates
Dedupe::Core::Kernel::FindFilenameDuplicatesExtern( Dedupe::FilePaths Paths,
                                                    bool recursiv )
{
  //we search for duplicates in the given orders, decide duplicates by filename
  return FindExternDuplicates( Paths, recursiv,
              [](Dedupe::FileInfo lhs, Dedupe::FileInfo rhs )
              { return lhs.GetPath().filename() > rhs.GetPath().filename();},

              [](Dedupe::FileInfo lhs, Dedupe::FileInfo rhs )
              { return lhs.GetPath().filename() == rhs.GetPath().filename();});
}

Dedupe::Core::Duplicates
Dedupe::Core::Kernel::FindFilesizeDuplicates()
{
  //we search for duplicates in the database, decide duplicates by filesize
  return FindDuplicates( Database.GetFiles(),
              [](Dedupe::FileInfo lhs, Dedupe::FileInfo rhs )
              { return lhs.GetSize() > rhs.GetSize();},

              [](Dedupe::FileInfo lhs, Dedupe::FileInfo rhs )
              { return lhs.GetSize() == rhs.GetSize();});
}


Dedupe::Core::Duplicates
Dedupe::Core::Kernel::FindFilesizeDuplicatesExtern( Dedupe::FilePaths Paths,
                                                    bool recursiv )
{
  //we search for duplicates in the given orders, decide duplicates by filesize
  return FindExternDuplicates( Paths, recursiv,
              [](Dedupe::FileInfo lhs, Dedupe::FileInfo rhs )
              { return lhs.GetSize() > rhs.GetSize();},

              [](Dedupe::FileInfo lhs, Dedupe::FileInfo rhs )
              { return lhs.GetSize() == rhs.GetSize();});
}

Dedupe::Core::Duplicates
Dedupe::Core::Kernel::FindFiledateDuplicates()
{
  //we search for duplicates in the database, decide duplicates by filedate
  return FindDuplicates( Database.GetFiles(),
              [](Dedupe::FileInfo lhs, Dedupe::FileInfo rhs )
              { return lhs.GetDateChanged() > rhs.GetDateChanged();},

              [](Dedupe::FileInfo lhs, Dedupe::FileInfo rhs )
              { return lhs.GetDateChanged() == rhs.GetDateChanged();});
}


Dedupe::Core::Duplicates
Dedupe::Core::Kernel::FindFiledateDuplicatesExtern( Dedupe::FilePaths Paths,
                                                    bool recursiv )
{
  //we search for duplicates in the given orders, decide duplicates by filedate
  return FindExternDuplicates( Paths, recursiv,
              [](Dedupe::FileInfo lhs, Dedupe::FileInfo rhs )
              { return lhs.GetDateChanged() > rhs.GetDateChanged();},

              [](Dedupe::FileInfo lhs, Dedupe::FileInfo rhs )
              { return lhs.GetDateChanged() == rhs.GetDateChanged();});
}

Dedupe::Core::Duplicates
Dedupe::Core::Kernel::FindDuplicates( Dedupe::FileStream CurrentFiles,
         std::function<bool(Dedupe::FileInfo, Dedupe::FileInfo)> SortFunc,
         std::function<bool(Dedupe::FileInfo, Dedupe::FileInfo)> EqualFunc)
{
  Duplicates Dups;
  HandleDuplicates Handle = 0;

  //if there is no file or one file, there can't be duplicates
  if( CurrentFiles.size() <= 1u ) return Dups;

  //Sort elements by hash value
  std::sort(CurrentFiles.begin(),
            CurrentFiles.end(),
            SortFunc);


  //We need two iterators to work
  Dedupe::FileStream::iterator WalkerBegin( CurrentFiles.begin() ),
                               WalkerEnd( WalkerBegin );

  Dedupe::Core::DuplicateGroup DupGroup;

  //this is the main search loop
  while( WalkerBegin != CurrentFiles.end() )
  {
    //Pos End to second element
    WalkerEnd = WalkerBegin + 1;

    /*increment End until an other hash is found
      also check if the end of the container is reached
      to prefent from a endless loop*/
    while( WalkerEnd != CurrentFiles.end()
           && EqualFunc( *WalkerBegin, *WalkerEnd ))
    {
      ++WalkerEnd;
    }

    //if only one element is found overjump, we search for duplicate groups
    if( (WalkerEnd - WalkerBegin) <= 1u )
    {
      ++WalkerBegin;

      continue;
    }

    while( WalkerBegin != WalkerEnd )
    {
      Handle = Dedupe::Core::HandleDuplicates::Empty;
      DupGroup.push_back( *WalkerBegin);
      ++WalkerBegin;
    }
    Dups.push_back( DupGroup );
    //Clear for next round in loop
    DupGroup.clear();

    WalkerBegin = WalkerEnd;
  }

  return Dups;
}

Dedupe::Core::Duplicates
Dedupe::Core::Kernel::FindExternDuplicates(
  Dedupe::FilePaths Paths,
  bool recursiv,
  std::function<bool(Dedupe::FileInfo, Dedupe::FileInfo)>SortFunc,
  std::function<bool(Dedupe::FileInfo, Dedupe::FileInfo)> EqualFunc)
{

  //Give founded files to FindDuplicates function
  return FindDuplicates( FileAndDirectoryHandler(Paths, recursiv),
                         SortFunc,
                         EqualFunc );
}

void Dedupe::Core::Kernel::ProcessDuplicates( FilesToProcess Dups )
{
  /*boost::system::error_code ec;

  for( auto it = Dups.begin(); it != Dups.end(); ++it )
  {
    for( auto InIt= it->begin(); InIt != it->end(); ++InIt )
    {
      switch( InIt->second() )
      {
        case Dedupe::Core::HandleDuplicates::Empty : continue; break;

        case Dedupe::Core::HandleDuplicates::Keep  : continue; break;

        case Dedupe::Core::HandleDuplicates::MarkAsKeep :

        InIt->first.SetKeep( true );
        Database.UpdateFile( InIt->first);
        break;

        case Dedupe::Core::HandleDuplicates::Delete:

        MessageOut << "Deleting file: " << InIt->first.GetPath() << std::endl;
        Database.DelFile( InIt->first );
        boost::filesystem::remove( InIt->first.GetPath(),ec);
        if( ec != 0 ) MessageOut << ec.message();
        break;

        default : MessageOut << "Request can't be handled: Unknown Token\n";
      }
    }
  }*/
}

void Dedupe::Core::Kernel::AutoUpdateDatabase()
{
  MessageOut << "Starting database update...\n";
  //Get a copy from the Files in the database
  Dedupe::FileStream FilesFromDatabase = Database.GetFiles();
  Dedupe::FileStream FilesToCheck;

  /*Use paths from the stored files to build
    new FileInfos and store them in FilesToCheck*/
  std::for_each( FilesFromDatabase.begin(),
                 FilesFromDatabase.end(),
        [ &FilesToCheck ]( Dedupe::FileInfo Current )
        { FilesToCheck.push_back( Dedupe::FileInfo( Current.GetPath())); });

  size_t i = 0;
  std::for_each( FilesToCheck.begin(),
                 FilesToCheck.end(),
        [ &FilesToCheck, FilesFromDatabase,this,&i ] (Dedupe::FileInfo Current )
        {
          /*if the filestatus is not FileOK delete file from database else
            update the file in database*/
          if( Current.GetStatus() != Dedupe::FileInfo::FileOK )
          {
            MessageOut << "Deleting file from database: "
                       << Current.GetPath() << std::endl;
            Database.DelFile( Current );
          }
          else
          {
            /*Update file only, if something of file has changed,
              but no check for hash*/
            if( !EqualNoHash( Current, FilesFromDatabase[ i ] ))
            {
              MessageOut << "Updating file in database: "
                         << Current.GetPath() << std::endl;
              Hasher.HashFileInfo( Current );
              Database.UpdateFile( Current );
            }
          }
          ++i;
        });

  MessageOut << "Databaseupdate finished.\n";
}