@Test
@Ignore
public void storeFile() throws IOException {
    File file = new File("/home/darrenfu/IdeaProjects/pampas/pampas-grpc/df/open/grpc/hello/grpc-test-229014610914606914.jar");
    GridFS gridFS = new GridFS(datastore.getDB());

    GridFSInputFile gridFSInputFile = gridFS.createFile(file);
    if (gridFS.findOne(file.getName()) == null) {
        gridFSInputFile.setId(file.getName());
        gridFSInputFile.setMetaData(new BasicDBObject("version", "1.1.2"));
        gridFSInputFile.save();
    }

    GridFSDBFile fsdbFile = gridFS.findOne(file.getName());
    File newfile = new File("/home/darrenfu/IdeaProjects/pampas/pampas-grpc/df/open/grpc/hello/grpc-test-229014610914606914.new.jar");
    if (newfile.exists()) {
        newfile.delete();
    }
    newfile.createNewFile();
    newfile.setWritable(true);

    fsdbFile.writeTo(newfile);
    System.out.println("done : " + fsdbFile.getFilename());
}
 

@Override
public long getEstimatedSizeBytes(PipelineOptions options) throws Exception {
  Mongo mongo = spec.connectionConfiguration().setupMongo();
  try {
    GridFS gridfs = spec.connectionConfiguration().setupGridFS(mongo);
    DBCursor cursor = createCursor(gridfs);
    long size = 0;
    while (cursor.hasNext()) {
      GridFSDBFile file = (GridFSDBFile) cursor.next();
      size += file.getLength();
    }
    return size;
  } finally {
    mongo.close();
  }
}
 

@Override
public List<GridFSDBFile> findAll(OrderBy orderBy, Page page) {
    DBCursor _cursor = __dbCollection.find();
    if (orderBy != null) {
        _cursor.sort(orderBy.toBson());
    }
    if (page != null && page.page() > 0 && page.pageSize() > 0) {
        _cursor.skip((page.page() - 1) * page.pageSize()).limit(page.pageSize());
    }
    List<GridFSDBFile> _results = new ArrayList<GridFSDBFile>();
    while (_cursor.hasNext()) {
        _results.add((GridFSDBFile) _cursor.next());
    }
    _cursor.close();
    return _results;
}
 

@Override
public List<GridFSDBFile> find(Query query, OrderBy orderBy, Page page) {
    DBCursor _cursor = __dbCollection.find(query.toBson());
    if (orderBy != null) {
        _cursor.sort(orderBy.toBson());
    }
    if (page != null && page.page() > 0 && page.pageSize() > 0) {
        _cursor.skip((page.page() - 1) * page.pageSize()).limit(page.pageSize());
    }
    List<GridFSDBFile> _results = new ArrayList<GridFSDBFile>();
    while (_cursor.hasNext()) {
        _results.add((GridFSDBFile) _cursor.next());
    }
    _cursor.close();
    return _results;
}
 

private boolean readFile(GridFSDBFile dbFile, OutputStream os) throws IOException {
	InputStream is = null;
	try {
		is = dbFile.getInputStream();
		byte data[] = new byte[4096];
		int len = 0;
		while ((len = is.read(data, 0, data.length)) > 0) {
			os.write(data, 0, len);
		}
		return true;
	} finally {
		if (is != null) {
			is.close();
		}
	}
}
 

public void sendFile(String path, HttpServletResponse response) throws IOException {
	GridFSDBFile file = getFile(path);
	if (file == null) {
		response.sendError(HttpServletResponse.SC_BAD_REQUEST, String.format("File %s does not exist", path));
	} else {
		String contentType = file.getContentType();
		if (contentType != null) {
			response.setContentType(contentType);
		}
		OutputStream os = null;
		try {
			readFile(file, os = response.getOutputStream());
		} catch (Exception e) {
			System.err.println("Send error: " + path);
		} finally {
			if (os != null) {
				os.close();
			}
		}
	}
}
 

/**
 * Converts a GridFSDBFile object into a struct
 * 
 * @param filefs
 * @return
 */
protected cfStructData	toStruct( GridFSDBFile filefs ){
	cfStructData	s	= new cfStructData();
	if ( filefs == null )
		return s;
	
	s.put("_id", 				new cfStringData(filefs.getId().toString()) );
	s.put("chunkSize", 	new cfNumberData(filefs.getChunkSize()) );
	s.put("length", 		new cfNumberData(filefs.getLength()) );
	s.put("md5",			 	new cfStringData(filefs.getMD5()) );
	s.put("filename", 	new cfStringData(filefs.getFilename()) );
	s.put("contentType", new cfStringData(filefs.getContentType()) );
	s.put("uploadDate", new cfDateData(filefs.getUploadDate()) );
	
	s.put("metadata", 	tagUtils.convertToCfData( filefs.getMetaData() ) );
	
	return s;
}
 

public static byte[] readFileFromGridFS(GridFSDBFile file) {
	InputStream is = file.getInputStream();
	ByteArrayOutputStream baos = new ByteArrayOutputStream();
	try {

		byte bytes[] = new byte[BUFFER_SIZE];
		int read = -1;

		while ((read = is.read(bytes)) != -1) {
			baos.write(bytes, 0, read);
		}

		return baos.toByteArray();
	}
	catch (Exception e) {
	}
	return null;
}
 

private AttachmentData fileToAttachmentData(final GridFSDBFile attachmant) {
  if (attachmant == null) {
    return null;
  } else {
    return new AttachmentData() {

      @Override
      public InputStream getInputStream() throws IOException {
        return attachmant.getInputStream();
      }

      @Override
      public long getSize() {
        return attachmant.getLength();
      }
    };
  }
}
 

@Override
public List<? extends BoundedSource<ObjectId>> split(
    long desiredBundleSizeBytes, PipelineOptions options) throws Exception {
  Mongo mongo = spec.connectionConfiguration().setupMongo();
  try {
    GridFS gridfs = spec.connectionConfiguration().setupGridFS(mongo);
    DBCursor cursor = createCursor(gridfs);
    long size = 0;
    List<BoundedGridFSSource> list = new ArrayList<>();
    List<ObjectId> objects = new ArrayList<>();
    while (cursor.hasNext()) {
      GridFSDBFile file = (GridFSDBFile) cursor.next();
      long len = file.getLength();
      if ((size + len) > desiredBundleSizeBytes && !objects.isEmpty()) {
        list.add(new BoundedGridFSSource(spec, objects));
        size = 0;
        objects = new ArrayList<>();
      }
      objects.add((ObjectId) file.getId());
      size += len;
    }
    if (!objects.isEmpty() || list.isEmpty()) {
      list.add(new BoundedGridFSSource(spec, objects));
    }
    return list;
  } finally {
    mongo.close();
  }
}
 

@Nullable
private InputStream getStreamByFilename(String fname)
{
    GridFSDBFile file = getGridFsTemplate().findOne(new Query().addCriteria(Criteria.where("filename").is(fname)));

    return file != null ? file.getInputStream() : null;
}
 

@GET
@Path("/{id}")
public Response getAttachment(@PathParam("id") final String id) throws IOException {
	final GridFS gridFS = new GridFS(this.mongoLegacyGrid);
	final GridFSDBFile file = gridFS.findOne(id);
	// log.info(file);
	if (file == null) {
		throw new WebApplicationException(404);
	}
	return Response.ok(org.apache.commons.io.IOUtils.toByteArray(file.getInputStream()), file.getContentType()).build();

}
 

private InputStream getMapFileByAppId(GridFS gridFS,String appIdentifier) throws MapFileNotExistsException{
	InputStream fileMap=null;
	GridFSDBFile gridFSFileMap = gridFS.findOne(appIdentifier);
	if(gridFSFileMap==null){
		throw new MapFileNotExistsException();
	}
	fileMap=gridFSFileMap.getInputStream();
	return fileMap;
}
 

private cfArrayData toArray( List<GridFSDBFile> list ) throws cfmRunTimeException{
	cfArrayData arr	= cfArrayData.createArray(1);
	if ( list == null || list.size() == 0 )
		return arr;
	
	Iterator<GridFSDBFile> it	= list.iterator();
	while ( it.hasNext() ){
		GridFSDBFile f	= it.next();
		arr.addElement( toStruct(f) );
	}
	
	return arr;
}
 

private MongoRepositoryItem createRepositoryItem(GridFSDBFile dbFile) {

    MongoRepositoryItem item = new MongoRepositoryItem(this, dbFile);

    Map<String, String> metadata = new HashMap<>();
    DBObject object = dbFile.getMetaData();
    for (String key : object.keySet()) {
      metadata.put(key, object.get(key).toString());
    }
    item.setMetadata(metadata);
    return item;
  }
 

private List<RepositoryItem> findRepositoryItemsByQuery(String query) {
  List<GridFSDBFile> files = gridFS.find((DBObject) JSON.parse(query));

  List<RepositoryItem> repositoryItems = new ArrayList<>();
  for (GridFSDBFile file : files) {
    repositoryItems.add(createRepositoryItem(file));
  }

  return repositoryItems;
}
 

private InputStream getOriginalInputStream(){
    DBObject query = new BasicDBObject(BuguFS.FILENAME, filename);
    query.put(DIMENSION, null);
    BuguFS fs = BuguFSFactory.getInstance().create(connection, bucket, chunkSize);
    GridFSDBFile f = fs.findOne(query);
    return f.getInputStream();
}
 

public BuguFS(String connectionName, String bucket, int chunkSize){
    this.bucket = bucket;
    this.chunkSize = chunkSize;
    DB db = BuguFramework.getInstance().getConnection(connectionName).getDB();
    fs = new GridFS(db, bucket);
    files = db.getCollection(bucket + ".files");
    //ensure the DBCursor can be cast to GridFSDBFile
    files.setObjectClass(GridFSDBFile.class);
}
 

public void rename(GridFSDBFile file, String newName){
    ObjectId id = (ObjectId)file.getId();
    DBObject query = new BasicDBObject(Operator.ID, id);
    DBObject dbo = files.findOne(query);
    dbo.put(FILENAME, newName);
    files.save(dbo);
}
 

private List<GridFSDBFile> toFileList(DBCursor cursor){
    List<GridFSDBFile> list = new ArrayList<GridFSDBFile>();
    while(cursor.hasNext()){
        DBObject dbo = cursor.next();
        list.add((GridFSDBFile)dbo);
    }
    cursor.close();
    return list;
}
 

@Override
public AttachmentMetadata getMetadata(AttachmentId attachmentId) throws IOException {
  final GridFSDBFile metadata = metadataGrid.findOne(attachmentId.serialise());

  if (metadata == null) {
    return null;
  }
  AttachmentProto.AttachmentMetadata protoMetadata =
      AttachmentProto.AttachmentMetadata.parseFrom(metadata.getInputStream());
  return new AttachmentMetadataProtoImpl(protoMetadata);
}
 

@Override
public GridFSDBFile findFirst(String filename) {
    return __gridFS.findOne(filename);
}
 

@Override
public GridFSDBFile findFirst(Query query) {
    return __gridFS.findOne(query.toBson());
}
 

@Override
public GridFSDBFile findById(String id) {
    return __gridFS.find(new ObjectId(id));
}
 

@Override
public List<GridFSDBFile> findAll() {
    return findAll(null, null);
}
 

@Override
public List<GridFSDBFile> findAll(OrderBy orderBy) {
    return findAll(orderBy, null);
}
 

@Override
public List<GridFSDBFile> find(String filename) {
    return __gridFS.find(filename);
}
 

@Override
public List<GridFSDBFile> find(String filename, OrderBy orderBy) {
    return __gridFS.find(filename, orderBy.toBson());
}
 

@Override
public List<GridFSDBFile> find(Query query) {
    return __gridFS.find(query.toBson());
}
 

@Override
public List<GridFSDBFile> find(Query query, OrderBy orderBy) {
    return __gridFS.find(query.toBson(), orderBy.toBson());
}