package org.kit.furia;
   
   import hep.aida.bin.StaticBin1D;
   
   import java.io.BufferedReader;
   import java.io.File;
   import java.io.FileReader;
   import java.io.IOException;
   import java.text.DecimalFormat;
  import java.text.NumberFormat;
  import java.util.Iterator;
  import java.util.List;
  
  import org.ajmm.obsearch.asserts.OBAsserts;
  import org.ajmm.obsearch.exception.NotFrozenException;
  import org.ajmm.obsearch.exception.OBException;
  import org.ajmm.obsearch.index.IndexFactory;
  import org.ajmm.obsearch.index.IndexShort;
  import org.ajmm.obsearch.index.PPTreeShort;
  import org.ajmm.obsearch.index.UnsafePPTreeShort;
  import org.ajmm.obsearch.index.pivotselection.AcceptAll;
  import org.ajmm.obsearch.index.pivotselection.KMeansPPPivotSelector;
  import org.apache.log4j.Logger;
  import org.kit.furia.exceptions.IRException;
  import org.kit.furia.fragment.OBFragment;
  import org.kit.furia.index.FIRIndexShort;
  import org.kit.furia.io.FuriaInputOBFragment;
  
  import com.sleepycat.je.DatabaseException;
  
  /*
   Furia-chan: An Open Source software license violation detector.    
   Copyright (C) 2008 Kyushu Institute of Technology
  
   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.
  
   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.
  
   You should have received a copy of the GNU General Public License
   along with this program.  If not, see <http://www.gnu.org/licenses/>.
   */
  
  /**
   * FuriaChanEngine holds the logic necessary to open fragmented programs insert
   * them, and search them in the database.
   * @author Arnoldo Jose Muller Molina
   */
  
  public class FuriaChanEngine {
  
      private static final Logger logger = Logger.getLogger("FuriaChanEngine");
  
      /**
       * Folder name where OB will reside.
       */
      protected static String OBSEARCH_FOLDER = "obsearch";
  
      protected static String IRINDEX_FOLDER = "irindex";
  
      /**
       * OBsearch index of our database.
       */
  
      protected IndexShort < OBFragment > index;
  
      /**
       * Multi-set index. The information retrieval engine that calculates the
       * similarity on documents or multi-sets of OBFragments.
       */
      protected IRIndexShort < OBFragment > mIndex;
  
      /**
       * The number of elements to get from OBSearch.
       */
      private byte k = 1;
  
      /**
       * Range to use for the tree distance function.
       */
      private short r = 7;
  
      /**
       * Get the top n elements.
       */
      private short n = 10;
  
      /**
       * Validation mode assumes that the query's document names are the same
       * names as in the DB. Based on this, it creates a summary of the queries
       * whose corresponding documents were found in the DB. For this to make
       * sense, all the apps in the DB should be different. Adding different
       * versions of a program might not give the indented effect.
       */
     private boolean validationMode = false;
 
     public boolean isValidationMode() {
         return validationMode;
     }
 
     public void setValidate(boolean validationMode) {
         this.validationMode = validationMode;
     }
 
     /**
      * Creates a FuriaChan object. If the given directory does not exist, the
      * directory will be created and two folders (one for OBSearch and one for
      * Lucene) will be created beneath it. If the directory exists, then the
      * corresponding OBSearch index and the IRIndex index will be loaded.
      * @param directory
      *                the database directory that will be used.
      * @throws IOException
      *                 If directory does not exist and it cannot be created.
      * @throws OBException
      * @throws InstantiationException
      * @throws IllegalAccessException
      * @throws NotFrozenException
      */
     public FuriaChanEngine(File directory) throws IOException,
             DatabaseException, NotFrozenException, IllegalAccessException,
             InstantiationException, OBException {
         File obFolder = new File(directory, OBSEARCH_FOLDER);
         File irFolder = new File(directory, IRINDEX_FOLDER);
         if (!directory.exists()) {
             directoryCreation(directory);
             directoryCreation(obFolder);
             directoryCreation(irFolder);
             index = createIndex(obFolder);
         } else { // load OBsearch and IRIndex
             OBAsserts.chkFileExists(obFolder);
             OBAsserts.chkFileExists(irFolder);
             IndexFactory < OBFragment > ifac = new IndexFactory < OBFragment >();
             if (ifac.isFrozen(obFolder)) {
                 index = (UnsafePPTreeShort < OBFragment >) ifac
                         .createFromOBFolder(obFolder);
                 index.relocateInitialize(null);
             } else { // the index is not frozen
                 index = createIndex(obFolder);
             }
         }
         mIndex = new FIRIndexShort < OBFragment >(index, irFolder);
     }
 
     public void close() throws IRException {
         mIndex.close();
     }
 
     /**
      * Freeze the index. After the index is frozen, matches can be performed.
      * @throws IRException
      *                 If there is an error in the freezing process.
      */
     public void freeze() throws IRException {
         mIndex.freeze();
     }
 
     /**
      * Insert the given directory into Furia-chan We will ignore applications
      * that have less than
      * @param dir
      *                The dir of an application or a directory of applications.
      * @throws IOException
      * @throws IRException
      */
     public void insert(File dir) throws IOException, IRException {
         FuriaInputOBFragment reader = new FuriaInputOBFragment(dir);
         Iterator < Document < OBFragment >> it = reader
                 .getDocumentsFromDirectory();
         while (it.hasNext()) {
             long prevTime = System.currentTimeMillis();
             Document < OBFragment > toAdd = it.next();
 
             if (toAdd.size() >= FuriaChanConstants.MIN_DOC_SIZE) {
                 mIndex.insert(toAdd);
                 logger.info("Loaded: " + toAdd.getName() + " size: "
                         + toAdd.size() + " msec: "
                         + (System.currentTimeMillis() - prevTime));
             } else {
                 logger.info("Document " + toAdd.getName()
                         + " was ignored because it is too small.");
             }
         }
     }
 
     /**
      * Performs a search in the database and prints the result to the user.
      * @param dir
      * @return The FuriaPrecision value (queries found in the top n results /
      *         total of queries)
      * @throws IOException
      * @throws IRException
      */
     public float search(File dir) throws IOException, IRException {
         logger.debug("Starting search with n:" + n + " r: " + r + " k: " + k
                 + " validation: " + this.validationMode + " msetThreshold "
                 + mIndex.getMSetScoreThreshold() + " setThreshold "
                 + mIndex.getSetScoreThreshold());
         FuriaInputOBFragment reader = new FuriaInputOBFragment(dir);
         Iterator < Document < OBFragment >> it = reader
                 .getDocumentsFromDirectory();
         int foundResults = 0; // only meaningful in validationMode
         int totalDocs = 0;
         // ********************************************************
         // The following applies to queries that are found within n top docs
         StaticBin1D setScoreStats = new StaticBin1D(); // statistics on sets
         // scores.
         StaticBin1D mSetScoreStats = new StaticBin1D(); // statistics on
         // multi-sets scores.
         StaticBin1D nStats = new StaticBin1D(); // statistics on n
         StaticBin1D objectsPerSecond = new StaticBin1D(); // statistics on n
         int maxSizeOfAppsNotFound = 0;
         StaticBin1D maxSizeStatsOfAppsNotFound = new StaticBin1D();
         // The following are stats that apply only for real answers found after
         // n
         StaticBin1D notMatchedMSet = new StaticBin1D(); // MSet score when the
         // query is not in n
         StaticBin1D notMatchedSet = new StaticBin1D(); // Set score when the
         // query is not in n
         StaticBin1D notMatchedN = new StaticBin1D(); // Position when the
         // result is not found
         // in n
         int notMatchedFountAfter = 0; // amount of not matched found after n.
         StaticBin1D completelyUnableToFindSize = new StaticBin1D(); // the guys
         // that even
         // after
         // extending
         // n can't
         // be found
         // The following are stats that apply only to apps that are not
         // candidates but are within the top
         // n results
         StaticBin1D notMatchedMSetWithinN = new StaticBin1D(); // MSet score
         // when the
         // query is not
         // in n
         StaticBin1D notMatchedSetWithinN = new StaticBin1D(); // Set score
         // when the
         // query is not
         // in n
         // The following are stats that apply only to the apps that are not the
         // candidate that
         // are found after n
         StaticBin1D notMatchedMSetAfterN = new StaticBin1D(); // MSet score
         // when the
         // query is not
         // in n
         StaticBin1D notMatchedSetAfterN = new StaticBin1D(); // Set score
         // when the
         // query is not
         // in n
         // ********************************************************
         logger.info("# of docs in the DB: " + this.mIndex.getSize());
         try {
             logger.info("# of words in the DB:" + this.mIndex.getWordsSize());
         } catch (DatabaseException d) {
             // :)
         }
         logger.info("(name, luceneScore, scoreMSet, scoreSet, size)");
         NumberFormat f = new DecimalFormat("0.000");
         short nToUse = n;
         int notFound = 0; // # of guys that were not even found in the list of
                             // candidates.
         if (this.validationMode) {
             mIndex.setValidationMode(true);
             nToUse = (short) (n + mIndex.getSize()); // used to get the
             // unmatched guys and
             // get statistics about
             // them
         }
         while (it.hasNext()) {
             Document < OBFragment > toSearch = it.next();
             if (validationMode) {
                 if (mIndex.shouldSkipDoc(toSearch)) {
                     logger.info("Validation mode: skipping:"
                             + toSearch.getName());
                     continue;
                 }
             }
             if (toSearch.size() >= FuriaChanConstants.MIN_DOC_SIZE) {
                 totalDocs++;
                 long prevTime = System.currentTimeMillis();
 
                 List < ResultCandidate > result = mIndex.search(toSearch, k, r,
                         nToUse);
                 float time = (float) (System.currentTimeMillis() - prevTime)
                         / (float) 1000;
                 logger.info("|| Match for " + toSearch.getName() + " sec:"
                         + time + " MSet: " + toSearch.multiSetSize() + " Set:"
                         + toSearch.size());
                 if (time > 0) {
                     objectsPerSecond.add((float) toSearch.size() / time);
                 }
                 Iterator < ResultCandidate > it2 = result.iterator();
                 int nth = 1;
                 boolean found = false;
 
                 logger.debug("Total results:" + result.size());
                 while (it2.hasNext() && nth <= this.n) {
                     ResultCandidate resultCandidate = it2.next();
                     String pre = "";
                     // hightlight the matched result.
                     if (validationMode
                             && resultCandidate.getDocumentName().equals(
                                     toSearch.getName())) {
                         foundResults++;
                         setScoreStats.add(resultCandidate.getNaiveScoreSet());
                         mSetScoreStats.add(resultCandidate.getNaiveScoreMSet());
                         nStats.add(nth);
                         found = true;
                         pre = "<<";
                     } else if (validationMode) {
                         notMatchedMSetWithinN.add(resultCandidate
                                 .getNaiveScoreMSet());
                         notMatchedSetWithinN.add(resultCandidate
                                 .getNaiveScoreSet());
                     }
 
                     logger.info(pre + resultCandidate.toString());
 
                     nth++;
                 }
 
                 // check if the item was found
                 if (validationMode && !found) {
                     if (maxSizeOfAppsNotFound < toSearch.size()) {
                         maxSizeOfAppsNotFound = toSearch.size();
                     }
                     maxSizeStatsOfAppsNotFound.add(toSearch.size());
                     boolean found2 = false;
                     while (it2.hasNext()) {
                         ResultCandidate resultCandidate = it2.next();
                         String docName = resultCandidate.getDocumentName();
                         // hightlight the matched result.
                         if (docName.equals(toSearch.getName())) {
                             found2 = true;
                             notMatchedMSet.add(resultCandidate
                                     .getNaiveScoreMSet());
                             notMatchedSet.add(resultCandidate
                                     .getNaiveScoreSet());
                             notMatchedN.add(nth);
                             logger.info(":(:(:( Found! pos: " + nth + " "
                                     + resultCandidate.toString());
                             break;
                         } else {
                             notMatchedMSetAfterN.add(resultCandidate
                                     .getNaiveScoreMSet());
                             notMatchedSetAfterN.add(resultCandidate
                                     .getNaiveScoreSet());
                         }
                         nth++;
                     }
                     if (!found2) {
                         completelyUnableToFindSize.add(toSearch.size());
                         logger.info(":(:(:(not found :( ");
                         notFound++;
                     }
                 }
             } else {
                 logger.warn(toSearch.getName()
                         + " ignored because it is too small");
             }
 
         }
         float result = ((float) foundResults / (float) totalDocs);
         // validationMode's summary
         if (validationMode) {
             
 
             printStats("MSet. Mean: ", mSetScoreStats);
             printStats("Set. Mean: ", setScoreStats);
             printStats("N. Mean: ", nStats);
             printStats("OBs per sec: ", objectsPerSecond);
             printStats("OBs not found (size). Mean: ",
                     maxSizeStatsOfAppsNotFound);
 
             printStats("Not matched (within N) MSet. Mean: ",
                     notMatchedMSetWithinN);
             printStats("Not matched (within N) Set. Mean: ",
                     notMatchedSetWithinN);
             printStats("Not matched (after N) MSet. Mean: ",
                     notMatchedMSetAfterN);
             printStats("Not matched (after N) Set. Mean: ", notMatchedSetAfterN);
             printStats(":(:(:(MSet. Mean: ", notMatchedMSet);
             printStats(":(:(:(Set. Mean: ", notMatchedSet);
             printStats(":(:(:(Nth. Mean: ", notMatchedN);
             printStats("Not in the results", completelyUnableToFindSize);
             logger.info("Not found count: " + notFound);
             logger
             .info("*** FuriaPrecision: (% of programs found in the first n documents) "
                     + result + " " + foundResults + " of " + totalDocs);
 
           
         }
         return result;
     }
 
     private void printStats(String msg, StaticBin1D stats) {
         logger.info(msg + " " + stats.mean() + " StdDev: "
                 + stats.standardDeviation() + " min: " + stats.min() + " max: "
                 + stats.max());
     }
 
     /**
      * Reads a String from the given file.
      * @param file
      *                File to Read
      * @return A String representation of the file
      * @throws IOException
      *                 If there is an IO error
      */
     public static String readString(final File file) throws IOException {
         final StringBuilder res = new StringBuilder();
         final BufferedReader metadata = new BufferedReader(new FileReader(file));
         String r = metadata.readLine();
         while (r != null) {
             res.append(r);
             r = metadata.readLine();
         }
         metadata.close();
         return res.toString();
     }
 
     /**
      * Utility class for creating the given directory
      * @param directory
      *                creates the given directory
      * @throws IOException
      *                 If the directory cannot be created
      */
     private void directoryCreation(File directory) throws IOException {
         directory.mkdirs();
         OBAsserts.chkFileExists(directory);
     }
 
     /**
      * A convenience method that creates an OBSearch index optimized for our
      * distance function.
      * @param folder
      * @return an optimized index
      */
     protected IndexShort < OBFragment > createIndex(File folder)
             throws IOException, DatabaseException {
 
         KMeansPPPivotSelector < OBFragment > ps = new KMeansPPPivotSelector < OBFragment >(
                 new AcceptAll < OBFragment >());
         ps.setRetries(1);
 
         return new UnsafePPTreeShort < OBFragment >(folder, (short) 30,
                 (byte) 12, (short) 0,
                 (short) (FuriaChanConstants.MAX_NODES_PER_FRAGMENT * 2), ps,
                 OBFragment.class);
     }
 
     public void setR(short r) {
         this.r = r;
     }
 
     public void setN(short n) throws OBException {
         OBAsserts.chkAssert(n > 0, "n should be greater than 0");
         this.n = n;
     }
 
     public void setK(byte k) {
         this.k = k;
     }
 
     public byte getK() {
         return k;
     }
 
     public short getR() {
         return r;
     }
 
     public short getN() {
         return n;
     }
 
     public void setValidationMode(boolean validationMode) {
         this.validationMode = validationMode;
     }
 
     public float getMSetScoreThreshold() {
         return mIndex.getMSetScoreThreshold();
     }
 
     public float getSetScoreThreshold() {
         return mIndex.getSetScoreThreshold();
     }
 
     public void setMSetScoreThreshold(float setScoreThreshold) {
         mIndex.setMSetScoreThreshold(setScoreThreshold);
     }
 
     public void setSetScoreThreshold(float setScoreThreshold) {
         mIndex.setSetScoreThreshold(setScoreThreshold);
     }
 
 }