private Map<String, Float> readDistanceScores(String startingObject, Set<String> others) {
	Map<String, Float> result = new HashMap<String, Float>();
	Query query = new org.springframework.data.mongodb.core.query.Query();
	query.addCriteria(Criteria.where("type.name").is(_SIMILARITY_METHOD).orOperator(
			Criteria.where("fromArtifact.$id").is(new ObjectId(startingObject)),
			Criteria.where("toArtifact.$id").is(new ObjectId(startingObject))));
	DBCollection dbCollection = mongoTemplate.getCollection("relation");
	DBCursor cursor = dbCollection.find(query.getQueryObject());
	List<DBObject> list = cursor.toArray();
	for (DBObject dbObject : list) {
		String toArtifact = ((DBRef) dbObject.get("toArtifact")).getId().toString();
		String fromArtifact = ((DBRef) dbObject.get("fromArtifact")).getId().toString();
		double value = ((double) dbObject.get("value"));
		if (toArtifact.equals(startingObject)) {
			if (others.contains(fromArtifact))
				result.put(fromArtifact, (float) (1 - value));
		} else if (others.contains(toArtifact))
			result.put(toArtifact, (float) (1 - value));
	}
	return result;
}
 

@Override
public void onApplicationEvent(MongoMappingEvent<?> event) {
    if (!(event instanceof BeforeSaveEvent)) {
        return;
    }
    if (!IdentifiableEntity.class.isAssignableFrom(event.getSource().getClass())) {
        return;
    }
    IdentifiableEntity source = (IdentifiableEntity) event.getSource();
    if (source.getId() != null) {
        return;
    }
    DBObject dbObject = event.getDBObject();
    String id = sequenceService.getNextIt(event.getSource().getClass());
    source.setId(id);
    dbObject.put("_id", id);
}
 

@Override
public void addElement(StringBuilder buf, DBObject result, Date date, Request request,
        Response response, long time) {
    // Don't need to flush since trigger for log message is after the
    // response has been committed
    long length = response.getBytesWritten(false);
    if (length <= 0) {
        // Protect against nulls and unexpected types as these values
        // may be set by untrusted applications
        Object start = request.getAttribute(
                Globals.SENDFILE_FILE_START_ATTR);
        if (start instanceof Long) {
            Object end = request.getAttribute(
                    Globals.SENDFILE_FILE_END_ATTR);
            if (end instanceof Long) {
                length = ((Long) end).longValue() -
                        ((Long) start).longValue();
            }
        }
    }
    if (length <= 0 && conversion) {
        result.put("bytesSent", '-');
    } else {
        result.put("bytesSent", length);
    }
}
 

public DBObject execute(){
	BasicDBObject plant = new BasicDBObject("code", getPlantData().getCode());
	DBObject thePlant = coll.findOne(plant);
	if (thePlant==null){
		coll.save(plant);
		System.out.println("New Plant has been saved into PlantCatalog DB");
		return plant;
	}

	System.out.println("in execute di PlantsCatalog: description->"  + getPlantData().getDescription());
	System.out.println("in execute di PlantsCatalog: price->"  + getPlantData().getPrice());

	coll.findAndModify(plant, thePlant);
		
	return thePlant;
}
 

public Pongo resolveReference(Object ref) {
	if (ref instanceof DBRef) {
		DBRef dbRef = (DBRef) ref;
	
		String fullyQualifiedId = dbRef.getDB().getName() + "." + dbRef.getRef() + "." + dbRef.getId().toString();
		Pongo pongo = (Pongo) cache.get(fullyQualifiedId);
		if (pongo == null) {
			DBObject dbObject = dbRef.getDB().getCollection(dbRef.getRef()).findOne(new BasicDBObject("_id", dbRef.getId()));
			if (dbObject != null) {
				pongo = createPongo(dbObject, dbRef.getDB().getCollection(dbRef.getRef()));
			}
		}
		return pongo;
	}
	else {
		return null;
	}
}
 

/**
 * Search the guild by given regex name
 * @param guildName
 * @return
 */
public final Collection<Guild> searchGuild(String guildName, int startPos, int count) {
	DBObject query = createDBObject();
	DBObject cond =createDBObject("$regex", guildName);
	cond.put("$options", "i");
	query.put(GUILDTITLE_NAME, cond);
	List<DBObject> dbObjs = null;
	if ( count > 0 ) {
		dbObjs = MongoDBUtil.queryAllFromMongo(
			query, databaseName, namespace, COLL_NAME, null, sorts, startPos, count);
	} else {
		dbObjs = MongoDBUtil.queryAllFromMongo(
			query, databaseName, namespace, COLL_NAME, null, sorts);
	}
	ArrayList<Guild> list = new ArrayList<Guild>();
	for ( DBObject dbObj : dbObjs ) {
		Guild guild = (Guild)MongoDBUtil.constructObject(dbObj);
		list.add(guild);
	}
	return list;
}
 

@SuppressWarnings("unchecked")
@Test
public void testCreateAggregate() {
    Map<String, Object> body = new HashMap<String, Object>();
    Map<String, Object> aggregate = new HashMap<String, Object>();
    Map<String, Object> assessments = new HashMap<String, Object>();
    Map<String, Object> mathTest = new HashMap<String, Object>();
    Map<String, Integer> highestEver = new HashMap<String, Integer>();
    highestEver.put("E", 15);
    highestEver.put("2", 20);
    mathTest.put("HighestEver", highestEver);
    assessments.put("ACT", mathTest);
    aggregate.put("assessments", assessments);
    DBObject dbObject = new BasicDBObjectBuilder().add("_id", "42").add("body", body)
            .add("aggregations", aggregate).get();
    CalculatedData<Map<String, Integer>> data = MongoEntity.fromDBObject(dbObject).getAggregates();
    assertEquals(Arrays.asList(new CalculatedDatum<Map<String, Integer>>("assessments", "HighestEver", "ACT",
            "aggregate", highestEver)), data.getCalculatedValues());
    
}
 

public DBIterator getIdCursorForFakeChemicals() {
  DBObject fakeRegex = new BasicDBObject();
  DBObject abstractInchi = new BasicDBObject();
  fakeRegex.put(ChemicalKeywords.INCHI$.MODULE$.toString(),
          new BasicDBObject(MongoKeywords.REGEX$.MODULE$.toString(), "^InChI=/FAKE"));

  abstractInchi.put(ChemicalKeywords.INCHI$.MODULE$.toString(),
          new BasicDBObject(MongoKeywords.REGEX$.MODULE$.toString(), "^InChI=.*R.*"));

  BasicDBList conditionList = new BasicDBList();
  conditionList.add(fakeRegex);
  conditionList.add(abstractInchi);

  BasicDBObject conditions = new BasicDBObject(MongoKeywords.OR$.MODULE$.toString(), conditionList);

  return getIteratorOverChemicals(conditions, new BasicDBObject(ChemicalKeywords.ID$.MODULE$.toString(), true));
}
 

private Map<String, Float> readDistanceScores(String object) {
	Map<String, Float> result = new HashMap<String, Float>();
	Query query = new org.springframework.data.mongodb.core.query.Query();
	query.addCriteria(Criteria.where("type.name").is(_SIMILARITY_METHOD).orOperator(
			Criteria.where("fromArtifact.$id").is(new ObjectId(object)),
			Criteria.where("toArtifact.$id").is(new ObjectId(object))));
	DBCollection dbCollection = mongoTemplate.getCollection("relation");
	DBCursor cursor = dbCollection.find(query.getQueryObject());
	List<DBObject> list = cursor.toArray();
	for (DBObject dbObject : list) {
		String toArtifact = ((DBRef) dbObject.get("toArtifact")).getId().toString();
		String fromArtifact = ((DBRef) dbObject.get("fromArtifact")).getId().toString();
		double value = ((double) dbObject.get("value"));
		if (toArtifact.equals(object))
			result.put(fromArtifact, (float) (1 - value));
		else
			result.put(toArtifact, (float) (1 - value));
	}
	return result;
}
 

@Test public void testPipelineQueryIsLimited() throws KettleException, MongoDbException {
  setupPerform();

  String query = "{$sort : 1}";

  // Setup DBObjects collection
  List<DBObject> dbObjects = new ArrayList<DBObject>();
  DBObject firstOp = (DBObject) JSON.parse( query );
  DBObject[] remainder = { new BasicDBObject( "$limit", NUM_DOCS_TO_SAMPLE ) };
  dbObjects.add( firstOp );
  Collections.addAll( dbObjects, remainder );
  AggregationOptions options = AggregationOptions.builder().build();

  //when( MongodbInputDiscoverFieldsImpl.jsonPipelineToDBObjectList( query ) ).thenReturn( dbObjects );
  when( collection.aggregate( anyList(), any( AggregationOptions.class ) ) )
      .thenReturn( cursor );

  discoverFields.discoverFields( new MongoProperties.Builder(), "mydb", "mycollection", query, "", true,
      NUM_DOCS_TO_SAMPLE, inputMeta );

  verify( collection ).aggregate( anyList(), any( AggregationOptions.class ) );
}
 

public static DBObject serialize(TransformedWaveletDelta transformedWaveletDelta) {
  BasicDBObject mongoTransformedWaveletDelta = new BasicDBObject();
  mongoTransformedWaveletDelta.append(FIELD_AUTHOR,
      serialize(transformedWaveletDelta.getAuthor()));
  mongoTransformedWaveletDelta.append(FIELD_RESULTINGVERSION,
      serialize(transformedWaveletDelta.getResultingVersion()));
  mongoTransformedWaveletDelta.append(FIELD_APPLICATIONTIMESTAMP,
      transformedWaveletDelta.getApplicationTimestamp());

  mongoTransformedWaveletDelta.append(FIELD_APPLIEDATVERSION,
      transformedWaveletDelta.getAppliedAtVersion());

  BasicDBList mongoWaveletOperations = new BasicDBList();

  for (WaveletOperation op : transformedWaveletDelta) {
    mongoWaveletOperations.add(serialize(op));
  }

  mongoTransformedWaveletDelta.append(FIELD_OPS, mongoWaveletOperations);

  return mongoTransformedWaveletDelta;
}
 

@Test
public void testRemoveDocument() {
	DBObject query = MongoDBUtil.createDBObject();
	int count = 10;
	for ( int i=0; i<count; i++ ) {
		DBObject dbObject = new BasicDBObject();
		dbObject.put("_id", i);
		dbObject.put("name", "value"+i);
		MongoDBUtil.saveToMongo(dbObject, dbObject, testDB, null, "mongoutil", true);
	}
	List<DBObject> list = MongoDBUtil.queryAllFromMongo(query, testDB, null, "mongoutil", null);
	assertTrue(list.size()>0);
	MongoDBUtil.removeDocument(testDB, null, "mongoutil", null);
	list = MongoDBUtil.queryAllFromMongo(query, testDB, null, "mongoutil", null);
	assertEquals(0, list.size());
}
 

/**
 * Delete an user from database by his/her name, including all the bag and 
 * relation data.
 * 
 * @param roleName
 * @return
 */
@Override
public void removeUserByRoleName(String roleName) {
	DBObject query = createDBObject(LOGIN_ROLENAME, roleName);
	DBObject field = createDBObject(_ID, 1);

	DBObject userObj = MongoDBUtil.queryFromMongo(query, databaseName, namespace, USER_COLL_NAME, field);
	if ( userObj == null ) {
		return;
	}
	byte[] bytes = (byte[])userObj.get(_ID);
	if ( bytes != null ) {
		UserId userId = UserId.fromBytes(bytes);
		this.removeUser(userId);
	} 
}
 

/**
 * get list of  the shards
 * @param dbConn
 * @return
 */
private static List<String> getShards(DB dbConn) {
    List<String> shards = new ArrayList<String>();

    DBObject listShardsCmd = new BasicDBObject("listShards", 1);
    CommandResult res = dbConn.command(listShardsCmd);
    if (!res.ok()) {
        LOG.error("Error getting shards for {}: {}", dbConn, res.getErrorMessage());
    }

    BasicDBList listShards = (BasicDBList) res.get("shards");

    //Only get shards for sharding mongo
    if (listShards != null) {
        ListIterator<Object> iter = listShards.listIterator();

        while (iter.hasNext()) {
            BasicDBObject shard = (BasicDBObject) iter.next();
            shards.add(shard.getString(ID));
        }
    }
    return shards;
}
 

public static void main(String[] args) throws Exception {

        MongoConnection mongo = new MongoConnection(MONGO_FT_CONNECTION);
        File ns_pmIds = new File(
                "/Users/richarde/Desktop/BBP_experiments/27_cleanup_pm-ft_in_mongo/CheckFtAgainstAbstracts.log.s.cut.uniq");
        int[] readInts = LineReader.intsFrom(new FileInputStream(ns_pmIds));

        for (int pmid : readInts) {

            DBObject r = new BasicDBObject("_id", pmid + "");
            mongo.coll.remove(r);
            Progress.tick();
        }

        System.err.println("done :-)");
    }
 

public void validate(DBObject sample){
	
	if(sample != null && sample.containsField(TARGET_FIELD_KEY)){
		int xValue = (Integer)sample.get(TARGET_FIELD_KEY);
		
		// Throw a standard exception if an illegal value is encountered
		if(xValue == illegalValue){
			throw new ServiceException("Illegal value for field_x", 
					SampleError.INVALID_SAMPLE).set(TARGET_FIELD_KEY, xValue);
		}
		
		// Change the value to clip to a configured maximum
		if(xValue > maxValue){
			sample.put(TARGET_FIELD_KEY, maxValue);
		}
	}
	else{
		// The field does not exist
		throw new ServiceException("Sample missing value for field_x", SampleError.INVALID_SAMPLE);
	}		
}
 

private void putMetadataInGridFS(boolean save) {
  DBObject metadataDBO = new BasicDBObject();
  for (Entry<String, String> entry : metadata.entrySet()) {
    metadataDBO.put(entry.getKey(), entry.getValue());
  }
  dbFile.setMetaData(metadataDBO);
  if (save) {
    dbFile.save();
  }
}
 

/**
 * If collection is very large, count() will be slow, you should use countFast().
 * @since mongoDB 3.4
 * @param key
 * @param value
 * @return 
 */
public long countFast(String key, Object value){
    long counter = 0;
    value = checkSpecialValue(key, value);
    Iterable<DBObject> results = aggregate().match(key, value).count("counter").results();
    Iterator<DBObject> it = results.iterator();
    if(it.hasNext()){
        DBObject dbo = it.next();
        String s = dbo.get("counter").toString();
        counter = Long.parseLong(s);
    }
    return counter;
}
 

@Override
public Object decode(Class targetClass, Object fromDBObject, MappedField optionalExtraInfo) {
    if (fromDBObject == null) return null;

    BasicDBList rawList = (BasicDBList) fromDBObject;

    AxisList axisList = new AxisList();
    for (Object obj : rawList) {
        DBObject dbObj = (DBObject) obj;
        axisList.add((Axis) getMapper().fromDBObject(optionalExtraInfo.getSubClass(), dbObj, getMapper().createEntityCache()));
    }

    return axisList;
}
 

public List<ChildResourceRef> getChildResourceRefs(String keyName, String keyValue) {

		ArrayList<ChildResourceRef> refList = new ArrayList<ChildResourceRef>();

		String now = LocalDateTime.now().toString(DateTimeFormat.forPattern("yyyyMMdd'T'HHmmss"));

		DBObject c1 = new BasicDBObject(EXPIRETIME_KEY, null);
		DBObject c2 = new BasicDBObject(EXPIRETIME_KEY, new BasicDBObject("$gt", now));
		
		BasicDBList or = new BasicDBList();
		or.add(c1);
		or.add(c2);
		
		BasicDBObject query = new BasicDBObject("$or", or).append(keyName,  keyValue);

		MongoCollection<Document> collection = context.getDatabaseManager()
				.getCollection(collectionName);
		//MongoCursor<Document> cursor = collection.find(
		//		new BasicDBObject(keyName, keyValue)).iterator();
		MongoCursor<Document> cursor = collection.find(query).iterator();

		while (cursor.hasNext()) {
			Document doc = cursor.next();
			
			ChildResourceRef ref = new ChildResourceRef();
			ref.setName(doc.getString(RESNAME_KEY));
			ref.setType(doc.getInteger(RESTYPE_KEY));
			ref.setValue(doc.getString(URI_KEY));
			
			refList.add(ref);
		}

		return refList;

	}
 

@Test
public void test() {
	final Merge merge = new Merge();
	final String[] builds = { "build1", "build2", "build3" };
	merge.builds = new ArrayList<String>();
	merge.builds.addAll(Arrays.asList(builds));

	when(this.cursor.next()).thenReturn(this.feature1, this.feature2, this.feature3);

	final DBObject result = BasicDBObject
			.parse("{'features' : [{'id' : 'f1', 'name' : 'f1', 'elements' : [ { 'id' : 'e1', 'name' : 'e1', 'statuses' : [ 'passed' , 'passed', 'failed', 'undefined']} , {'id' : 'e2', 'name' : 'e2', 'statuses' : ['failed', 'undefined' , 'failed', 'passed']}], 'statuses' : ['failed', 'undefined', 'failed', 'undefined'], 'url': 'reports/p1/1.1.1/{{BUILD_NAME}}/f1'}], 'builds':['Merged', 'build1', 'build2', 'build3']}");
	assertEquals(this.mergeBuilds.getMergedBuilds(merge), result);
}
 

public Map<String, Object> selectOne(String dsKey, String sql) {
	SelectVo selectVo = (SelectVo) sqlParser.parse(sql);
	DBCollection collection = MongoSupport.getCollection(dsKey, selectVo.getTable());
	DBObject result = selectVo.selectOne(collection);
	if (null != result) {
		return getResults(result, null);
	}
	return null;
}
 

private void log(DBObject fields, DBObject query, DBObject orderByObject) {
	if (log.isInfoEnabled()) {
		if (null != fields) {
			log.info("field:" + fields.toString());
		}
		if (null != query) {
			log.info("query:" + query.toString());
		}
		if (null != orderByObject) {
			log.info("order:" + orderByObject.toString());
		}
	}
}
 

private Long getEnzSummaryIDAsLong(BasicDBList reactant, int i) {
  try {
    return (Long)((DBObject)reactant.get(i)).get("pubchem");
  } catch (ClassCastException e) {
    return ((Integer)((DBObject)reactant.get(i)).get("pubchem")).longValue();
  }
}
 

private double stdDevSamp(String key, DBObject query){
    double result = 0;
    BuguAggregation agg = this.aggregate();
    agg.match(query);
    String json = "{_id:null, devValue:{$stdDevSamp:'$" + key + "'}}";
    agg.group(json);
    Iterator it = agg.results().iterator();
    if(it.hasNext()){
        DBObject dbo = (DBObject)it.next();
        result = (Double)dbo.get("devValue");
    }
    return result;
}
 

private static boolean getBooleanValueOfKey( DBObject commandObj, String keyName, boolean defaultValue )
{
    Object value = commandObj.get( keyName );
    if( value == null )
        return defaultValue;
    if( value instanceof Number )
        return ((Number)value).intValue() > 0;
    if( value instanceof Boolean )
        return ((Boolean)value).booleanValue();
    return defaultValue;
}
 

public List<Document> aggregate(String actualDate, String expectedDate) {
    DBObject pushFields = new BasicDBObject();
    pushFields.put("server", "$server");
    pushFields.put("branch", "$branch");
    pushFields.put("commit", "$commit");
    pushFields.put("testcaseId", "$testcaseId");
    pushFields.put("story", "$story");
    pushFields.put("state", "$state");
    pushFields.put("datetime", "$datetime");
    pushFields.put("elements", "$elements");
    pushFields.put("url", "$url");
    pushFields.put("device", "$device");
    pushFields.put("osName", "$osName");
    pushFields.put("osVersion", "$osVersion");
    pushFields.put("browserName", "$browserName");
    pushFields.put("browserVersion", "$browserVersion");
    pushFields.put("resolution", "$resolution");
    pushFields.put("retina", "$retina");

    List<Document> documents = collection.aggregate(asList(
            match(regex("_id", "^(" + actualDate + "|" + expectedDate + ")")),
            group("$hash", push("snapshot", pushFields))))
            .batchSize(500)
            .allowDiskUse(true)
            .into(new ArrayList<>());

    return documents;
}
 

public List<Document> find() {
    List<Bson> matchFilters = new ArrayList<>();

    DBObject groupFields = new BasicDBObject();
    groupFields.put("datetime", "$datetime");

    DBObject pushFields = new BasicDBObject();
    pushFields.put("server", "$server");
    pushFields.put("branch", "$branch");
    pushFields.put("commit", "$commit");
    pushFields.put("testcaseId", "$testcaseId");
    pushFields.put("story", "$story");
    pushFields.put("state", "$state");
    pushFields.put("url", "$url");

    List<Bson> pipeline = new ArrayList<>();
    if (matchFilters.size() > 0) {
        pipeline.add(match(and(matchFilters)));
    }
    pipeline.add(group(new BasicDBObject("_id", groupFields), push("snapshot", pushFields)));

    List<Document> documents = collection.aggregate(pipeline)
            .allowDiskUse(true)
            .into(new ArrayList<>());

    return documents;
}
 

/**
 * Query the GuildBag for given guild.
 * @param guildId
 * @return
 */
public GuildBag queryGuildBag(String guildId) {
	DBObject query = createDBObject();
	query.put(INDEX_NAME, guildId);
	DBObject dbObj = MongoDBUtil.queryFromMongo(query, databaseName, namespace, COLL_BAG_NAME, null);
	GuildBag guildBag = (GuildBag)MongoDBUtil.constructObject(dbObj);
	//Check the current count.
	if ( guildBag != null ) {
		if ( guildBag.getCount() != guildBag.getPropList().size() ) {
			guildBag.setCount(guildBag.getPropList().size());
		}
	}
	return guildBag;
}
 

protected WriteResult batchRetryUsingSave( boolean lastRetry )
  throws MongoException, KettleException, MongoDbException {
  WriteResult result = null;
  int count = 0;
  logBasic( BaseMessages.getString( PKG, "MongoDbOutput.Messages.CurrentBatchSize", m_batch.size() ) );
  for ( int i = 0, len = m_batch.size(); i < len; i++ ) {
    DBObject toTry = m_batch.get( i );
    Object[] correspondingRow = m_batchRows.get( i );
    try {
      result = m_data.getCollection().save( toTry );
      count++;
    } catch ( MongoException ex ) {
      if ( !lastRetry ) {
        logBasic( BaseMessages.getString( PKG, "MongoDbOutput.Messages.SuccessfullySavedXDocuments", count ) );
        m_batch = copyExceptFirst( count, m_batch );
        m_batchRows = copyExceptFirst( count, m_batchRows );
        throw ex;
      }

      // Send this one to the error stream if doing error handling
      if ( getStepMeta().isDoingErrorHandling() ) {
        putError( getInputRowMeta(), correspondingRow, 1, ex.getMessage(), "", "MongoDbOutput" );
      } else {
        m_batch = copyExceptFirst( i + 1, m_batch );
        m_batchRows = copyExceptFirst( i + 1, m_batchRows );
        throw ex;
      }
    }
  }

  m_batch.clear();
  m_batchRows.clear();

  logBasic( BaseMessages.getString( PKG, "MongoDbOutput.Messages.SuccessfullySavedXDocuments", count ) );

  return result;
}