Allow XWiki to scale with large number of objects

{{groovy}}
import java.util.UUID;
import java.util.Date;
import com.xpn.xwiki.doc.XWikiDocument;
wiki = xcontext.getContext().getWiki();
xc = xcontext.getContext();
String xcDb = xc.getDatabase();
XWikiDocument benchDoc;
if(!wiki.exists("Test.objectBench1", xc)){
    benchDoc = wiki.getDocument("Test.objectBench1", xc);
    for(int i=0; i<1000; i++){
        thisObj = benchDoc.newObject("XWiki.XWikiComments", xc);
        thisObj.setStringValue("author", UUID.randomUUID().toString());
        thisObj.setStringValue("highlight", UUID.randomUUID().toString());
        thisObj.setStringValue("comment", UUID.randomUUID().toString());
    }
    wiki.saveDocument(benchDoc, xc);
}else{
    benchDoc = new XWikiDocument();
    benchDoc.setFullName("Test.objectBench1", xc);
}
try{
    if(benchDoc.getDatabase() != null){xc.setDatabase(benchDoc.getDatabase());}
    startTime = new Date().getTime();
    benchDoc = wiki.getHibernateStore().loadXWikiDoc(benchDoc, xc);
    println("It took "+(new Date().getTime() - startTime)+"ms. to load a document with "+
                benchDoc.getxWikiObjects().get("XWiki.XWikiComments").size()+" comments.");
}finally{xc.setDatabase(xcDb);}
{{/groovy}}

{{groovy}}
import java.util.UUID;
import java.util.Date;
import java.util.Vector;
import com.xpn.xwiki.doc.XWikiDocument;
import com.xpn.xwiki.objects.BaseObject;
import com.xpn.xwiki.objects.BaseProperty;
import com.xpn.xwiki.objects.BaseStringProperty;
import com.xpn.xwiki.objects.PropertyInterface;
import org.hibernate.Query;
import org.hibernate.Session;
wiki = xcontext.getContext().getWiki();
xc = xcontext.getContext();
String xcDb = xc.getDatabase();
XWikiDocument benchDoc;
if(!wiki.exists("Test.objectBench1", xc)){
    benchDoc = wiki.getDocument("Test.objectBench1", xc);
    for(int i=0; i<1000; i++){
        thisObj = benchDoc.newObject("XWiki.XWikiComments", xc);
        thisObj.setStringValue("author", UUID.randomUUID().toString());
        thisObj.setStringValue("highlight", UUID.randomUUID().toString());
        thisObj.setStringValue("comment", UUID.randomUUID().toString());
    }
    wiki.saveDocument(benchDoc, xc);
}else{
    benchDoc = new XWikiDocument();
    benchDoc.setFullName("Test.objectBench1", xc);
}


Map<String, Vector<BaseObject>> xWikiObjects = new TreeMap<String, Vector<BaseObject>>();
//It takes a ridiculously long time to resize a Vector, so we will store the size
//until the end when we will set the sizes of all Vectors and copy in all objects.
Map<String, Integer> numberOfObjectsPerClass = new HashMap<String, Integer>();
try{

    startTime = new Date().getTime();

    newDoc = new XWikiDocument();
    newDoc.setFullName("Test.objectBench1", xc);
    //here is where we start the fast object getter
    //begin a transaction.
    hbs = wiki.getHibernateStore();
    hbs.checkHibernate(xc);
    hbs.beginTransaction(false, xc);
    Session session = hbs.getSession(xc);

    //get all objects associated with this document
    Query q = session.createQuery("select obj from BaseObject obj where obj.name = :name order by obj.id");
    q.setText("name", benchDoc.getFullName());
    List<BaseObject> objects = q.list();
    if(objects.size() == 0){return;}

    //get a list of all object id numbers
    List<Integer> objectIds = new ArrayList<Integer>(objects.size());
    for(BaseObject obj : objects){
        obj.setWiki(xc.getDatabase());
        objectIds.add(obj.getId());
    }

    //now get a single list of all properties associated with any one of the above id numbers
    q = session.createQuery("select prop from BaseProperty prop where prop.id in (:ids) order by prop.id");
    q.setParameterList("ids", objectIds);
    List<BaseProperty> props = q.list();

    //this is one of the slowest ways of sorting unordered data, however
    //the data was ordered by the database so it need only be interleaved.
    int objectNum = 0;
    BaseObject thisObject = objects.get(objectNum);
    int propNum = 0;
    BaseProperty thisProp = props.get(propNum);
    while(true){
        //because the data is ordered by id, once we find a property whose id 
        //doesn't match the id of the current object, we assume we are done adding
        //properties to that object and that our property will match the next object.
        if(thisProp.getId() == thisObject.getId()){
            if(thisProp.getValue() != null){
            }else if(thisProp instanceof BaseStringProperty){
                //Null value strings should be interpreted as ""
                thisObject.setValue((Object) "");
            }
            thisObject.safeput(thisProp.getName(), (PropertyInterface) thisProp);
            propNum++;
            //if we run out of properties, we are done.
            if(propNum == props.size()){
                //Set the sizes of the vectors.
                for(String className : xWikiObjects.keySet()){
                    xWikiObjects.get(className).setSize(numberOfObjectsPerClass.get(className));
                }
                //Copy the objects into the vectors
                for(BaseObject object : objects){
                    xWikiObjects.get(object.getClassName()).set(object.getNumber(), object);
                }
                //save the vector map to the document
                newDoc.setxWikiObjects(xWikiObjects);
                //save the current object and go home.
                newDoc.setObject(thisObject.getClassName(), thisObject.getNumber(), thisObject);
                break;
            }
            thisProp = props.get(propNum);
        }else{
            //We are out of properties with matching id, lets move on to the next object.
            //First we must save the object.
            //In order to preserve the order of classes in the map, we have to save
            //in the same order as loadXWikiDoc. loadXWikiDoc sorts by number, but all
            //all classes will be represented before the number increments from 0 so they
            //are really ordered by the secondary order which is the database default (the key)
            //We are explicitally sorting by key so order will be the same unless the db is corrupted
            //in that case order of classes in the tree map is a low priority.

            if(!numberOfObjectsPerClass.containsKey(thisObject.getClassName())){
                //If we discover a new class, add a placeholder, we will add the objects later
                xWikiObjects.put(thisObject.getClassName(), new Vector<BaseObject>());
                numberOfObjectsPerClass.put(thisObject.getClassName(), 1);
            }
            if(numberOfObjectsPerClass.get(thisObject.getClassName()) <= thisObject.getNumber()){
                //If the stored number of objects for this class is too small, store a larger number
                numberOfObjectsPerClass.put(thisObject.getClassName(), thisObject.getNumber()+1)
            }

            objectNum++;
            if(objectNum == objects.size()){
                //This should never happen, but if the data it out of order we loop through the
                //objects once again (until every property has found a home.)
                objectNum = 0;
                println("LOG: object numbers rolled over!");
            }
            thisObject = objects.get(objectNum);
        }
    }
    hbs.endTransaction(xc, false, false);
}finally{//xc.setDatabase(xcDb);
    try{
        hbs.endTransaction(xc, false, false);
    }catch(Exception e){}
}



println("It took "+(new Date().getTime() - startTime)+"ms. to load a document with "+
            newDoc.getObjects("XWiki.XWikiComments").size()+" comments.");



//now compare to the loadXWikiDoc copy to make sure it's accurate.
try{
    if(benchDoc.getDatabase() != null){xc.setDatabase(benchDoc.getDatabase());}
    benchDoc = wiki.getHibernateStore().loadXWikiDoc(benchDoc, xc);
}finally{xc.setDatabase(xcDb);}

Vector<BaseObject> comments = newDoc.getObjects("XWiki.XWikiComments");

if(comments.size() != 0){
    Vector<BaseObject> realObjects = benchDoc.getObjects("XWiki.XWikiComments");
    int failed = 0;
    for(int i=0; i<realObjects.size(); i++){
        if(i == comments.size()){
            println("not enough objects, there are "+realObjects.size()+" you only got "+comments.size());
            break;
        }
        if(!realObjects.get(i).equals(comments.get(i))){
            failed++;
            if(failed < 5){
                println("my object:   "+comments.get(i).getNumber());
                println("real object: "+realObjects.get(i).getNumber());
                for(String fieldName : realObjects.get(i).getPropertyList()){
                    if(!realObjects.get(i).getField(fieldName).equals(
                        comments.get(i).getField(fieldName))){
                        if(realObjects.get(i).getStringValue(fieldName).equals(
                                        comments.get(i).getStringValue(fieldName))){
                            println(fieldName: "(matches)");
                        }else{
                            println(fieldName+" "+realObjects.get(i).getStringValue(fieldName)+
                                " "+comments.get(i).getStringValue(fieldName));
                        }
                    }
                }
            }
        }
    }
    if(failed != 0){println("there were "+failed+" object mismatches.");}
}else{
    println("no objects were returned!");
}

{{/groovy}}

if (list2 == null && obj.getClass().getName().indexOf(".BaseProperty")!=-1){
    list2 = new ArrayList<String>();
}

if(thisProp.getClassType().indexOf(".DBStringListProperty")!=-1
   && ((List) thisProp.getValue()).size()==0){
    session.evict(thisProp);
    session.load(thisProp, thisProp);
}

#-# New more scalable document loader. The new loader is faster at loading documents which have lots of objects,
#-# however it is new and complex so it is turned off by default. To turn it on, uncomment the following line and set it to =1
# xwiki.store.hibernate.useNewDocumentLoader=0