@Override
public AppEngineDataStore<V> set(String key, V value) throws IOException {
  Preconditions.checkNotNull(key);
  Preconditions.checkNotNull(value);
  lock.lock();
  try {
    Entity entity = new Entity(getId(), key);
    entity.setUnindexedProperty(FIELD_VALUE, new Blob(IOUtils.serialize(value)));
    dataStoreService.put(entity);
    if (memcache != null) {
      memcache.put(key, value, memcacheExpiration);
    }
  } finally {
    lock.unlock();
  }
  return this;
}
 

@Test
public void testSize() {
    String kind = kindName + "-size";
    int recordSize = (1000 * 1000);  // Max. 1000000.
    byte[] filledRec = new byte[recordSize];
    Arrays.fill(filledRec, (byte) 0x41);
    Blob bigBlob = new Blob(filledRec);
    assertEquals(recordSize, bigBlob.getBytes().length);
    Entity eBlob = new Entity(kind, rootKey);
    eBlob.setProperty("blobProp", bigBlob);
    service.put(eBlob);

    recordSize = 500 ;  // Max. 500.
    filledRec = new byte[recordSize];
    Arrays.fill(filledRec, (byte) 0x41);
    ShortBlob shortBlob = new ShortBlob(filledRec);
    assertEquals(recordSize, shortBlob.getBytes().length);
    Entity eShortBlob = new Entity(kind, rootKey);
    eShortBlob.setProperty("byteStrProp", shortBlob);
    service.put(eShortBlob);

    service.delete(eBlob.getKey(), eShortBlob.getKey());
}
 

private static SimpleModule getWriteBlobAsBase64Module() {
  JsonSerializer<Blob> dateSerializer = new JsonSerializer<Blob>() {
    @Override
    public void serialize(Blob value, JsonGenerator jgen, SerializerProvider provider)
        throws IOException {
      byte[] bytes = value.getBytes();
      jgen.writeBinary(bytes, 0, bytes.length);
    }
  };
  SimpleModule writeBlobAsBase64Module = new SimpleModule("writeBlobAsBase64Module",
      new Version(1, 0, 0, null, null, null));
  writeBlobAsBase64Module.addSerializer(Blob.class, dateSerializer);
  return writeBlobAsBase64Module;
}
 

@Test
public void testReadBlobParameter() throws Exception {
  Method method =
      TestEndpoint.class.getDeclaredMethod("doBlob", Blob.class);
  Object[] params = readParameters("{\"blob\":\"AQIDBA==\"}", method);

  assertEquals(1, params.length);
  assertThat(((Blob) params[0]).getBytes()).isEqualTo(new byte[]{1, 2, 3, 4});
}
 

@SuppressWarnings("unused")
public void testBlobAsBase64() throws Exception {
  Object value = new Object() {
    public Blob getBlob() {
      return new Blob(new byte[]{1, 2, 3, 4});
    }
  };
  ObjectNode output = ObjectMapperUtil.createStandardObjectMapper()
      .readValue(writeToResponse(value), ObjectNode.class);
  assertEquals("AQIDBA==", output.path("blob").asText());
}
 

@Override
public Entity toEntity() {
  Entity entity = toProtoEntity();
  entity.setUnindexedProperty(SHARD_ID_PROPERTY, shardId);
  entity.setUnindexedProperty(VALUE_PROPERTY, new Blob(value));
  return entity;
}
 

public ExceptionRecord(Entity entity) {
  super(entity);
  Blob serializedExceptionBlob = (Blob) entity.getProperty(EXCEPTION_PROPERTY);
  byte[] serializedException = serializedExceptionBlob.getBytes();
  try {
    exception = (Throwable) SerializationUtils.deserialize(serializedException);
  } catch (IOException e) {
    throw new RuntimeException("Failed to deserialize exception for " + getKey(), e);
  }
}
 

@Override
public Entity toEntity() {
  try {
    Entity entity = toProtoEntity();
    byte[] serializedException = SerializationUtils.serialize(exception);
    entity.setUnindexedProperty(EXCEPTION_PROPERTY, new Blob(serializedException));
    return entity;
  } catch (IOException e) {
    throw new RuntimeException("Failed to serialize exception for " + getKey(), e);
  }
}
 

@Override
public Object serializeValue(PipelineModelObject model, Object value) throws IOException {
  byte[] bytes = SerializationUtils.serialize(value);
  if (bytes.length < MAX_BLOB_BYTE_SIZE) {
    return new Blob(bytes);
  }
  int shardId = 0;
  int offset = 0;
  final List<Entity> shardedValues = new ArrayList<>(bytes.length / MAX_BLOB_BYTE_SIZE + 1);
  while (offset < bytes.length) {
    int limit = offset + MAX_BLOB_BYTE_SIZE;
    byte[] chunk = Arrays.copyOfRange(bytes, offset, Math.min(limit, bytes.length));
    offset = limit;
    shardedValues.add(new ShardedValue(model, shardId++, chunk).toEntity());
  }
  return tryFiveTimes(new Operation<List<Key>>("serializeValue") {
    @Override
    public List<Key> call() {
      Transaction tx = dataStore.beginTransaction();
      List<Key> keys;
      try {
        keys = dataStore.put(tx, shardedValues);
        tx.commit();
      } finally {
        if (tx.isActive()) {
          tx.rollback();
        }
      }
      return keys;
    }
  });
}
 

static SessionData createSessionFromEntity(Entity entity) {
  SessionData data = new SessionData();
  data.setExpirationTime((Long) entity.getProperty(EXPIRES_PROP));

  Blob valueBlob = (Blob) entity.getProperty(VALUES_PROP);
  @SuppressWarnings("unchecked")
  Map<String, Object> valueMap = (Map<String, Object>) deserialize(valueBlob.getBytes());
  data.setValueMap(valueMap);
  return data;
}
 

/**
 * Return an {@link Entity} for the given key and data in the empty
 * namespace.
 */
static Entity createEntityForSession(String key, SessionData data) {
  String originalNamespace = NamespaceManager.get();
  try {
    NamespaceManager.set("");
    Entity entity = new Entity(SESSION_ENTITY_TYPE, key);
    entity.setProperty(EXPIRES_PROP, data.getExpirationTime());
    entity.setProperty(VALUES_PROP, new Blob(serialize(data.getValueMap())));
    return entity;
  } finally {
    NamespaceManager.set(originalNamespace);
  }
}
 

@Test
public void testBlobType() {
    String propertyName = "blobProp";
    List<Entity> elist = doQuery(kindName, propertyName, null, false);
    Blob blob = (Blob) elist.get(0).getProperty(propertyName);
    Blob sameDat = (Blob) elist.get(0).getProperty(propertyName);
    Blob diffDat = (Blob) elist.get(1).getProperty(propertyName);
    assertTrue(blob.equals(sameDat));
    assertFalse(blob.equals(diffDat));
    byte[] blobData = blob.getBytes();
    assertTrue(Arrays.equals("blobImage".getBytes(), blobData) ||
               Arrays.equals("blobText".getBytes(), blobData) ||
               Arrays.equals("blobData".getBytes(), blobData));
    assertEquals(blob.hashCode(), blob.hashCode());
}
 

@Test
public void testUnindexedProperties() throws Exception {
    Entity entity = new Entity(UNINDEXED_ENTITY);
    entity.setUnindexedProperty("unindexedString", "unindexedString");
    entity.setUnindexedProperty("unindexedList", new ArrayList<String>(Arrays.asList("listElement1", "listElement2", "listElement3")));
    entity.setUnindexedProperty("unindexedText", new Text("unindexedText"));
    entity.setUnindexedProperty("unindexedBlob", new Blob("unindexedBlob".getBytes()));
    entity.setProperty("text", new Text("text"));
    entity.setProperty("blob", new Blob("blob".getBytes()));

    Key key = service.put(entity);
    sync(3000);  // Not using ancestor queries, so pause before doing queries below.
    Entity entity2 = service.get(key);

    assertTrue(isUnindexed(getRawProperty(entity2, "unindexedString")));
    assertTrue(isUnindexed(getRawProperty(entity2, "unindexedList")));
    assertTrue(isUnindexed(getRawProperty(entity2, "unindexedText")));
    assertTrue(isUnindexed(getRawProperty(entity2, "unindexedBlob")));
    assertTrue(isUnindexed(getRawProperty(entity2, "text")));
    assertTrue(isUnindexed(getRawProperty(entity2, "blob")));

    assertNull(getResult(new Query(UNINDEXED_ENTITY).setFilter(new FilterPredicate("unindexedString", EQUAL, "unindexedString"))));
    assertNull(getResult(new Query(UNINDEXED_ENTITY).setFilter(new FilterPredicate("unindexedList", EQUAL, "listElement1"))));
    assertNull(getResult(new Query(UNINDEXED_ENTITY).setFilter(new FilterPredicate("unindexedText", EQUAL, "unindexedText"))));
    assertNull(getResult(new Query(UNINDEXED_ENTITY).setFilter(new FilterPredicate("text", EQUAL, "text"))));

    service.delete(key);
}
 

@Override
public Blob deserialize(JsonParser jsonParser, DeserializationContext context)
    throws IOException {
  return new Blob(jsonParser.getBinaryValue());
}
 

@Override
protected void service(HttpServletRequest req, HttpServletResponse resp)
    throws ServletException, IOException {
    Calendar cal = Calendar.getInstance();
    cal.set(Calendar.HOUR, 0);
    cal.set(Calendar.MINUTE, 0);
    cal.set(Calendar.SECOND, 0);
    cal.set(Calendar.MILLISECOND, 0);

    cal.add(Calendar.DATE, -10);

    Downloaders downloaders = new Downloaders();
    DataService data = new DataService();

    for (int i = 0; i < 10; i++) {
        List<Puzzle> puzzles = downloaders.getPuzzles(cal.getTime());
        System.out.println("Got " + puzzles.size() + " puzzles for " +
            cal.getTime());

        for (Puzzle puz : puzzles) {
            PuzzleListing listing = data.findPuzzleListingBySourceAndDate(puz.getSource(),
                    cal.getTime());

            if (listing != null) {
                System.out.println("Puzzle from " + puz.getSource() +
                    " already in database.");
            } else {
                System.out.println("Persisting from " + puz.getSource() +
                    ".");
                listing = new PuzzleListing();
                listing.setDate(cal.getTime());
                listing.setSource(puz.getSource());
                listing.setTitle(puz.getTitle());

                ByteArrayOutputStream puzData = new ByteArrayOutputStream();
                ByteArrayOutputStream meta = new ByteArrayOutputStream();
                IO.save(puz, new DataOutputStream(puzData), new DataOutputStream(meta));
                listing.setPuzzleSerial(new Blob(puzData.toByteArray()));
                listing.setMetaSerial(new Blob(meta.toByteArray()));
                data.store(listing);
            }
        }

        cal.add(Calendar.DATE, 1);
    }

    data.close();
    PuzzleServlet.CACHE.put("puzzle-list", null);
}
 

public void setContent( Blob content ) {
 this.content = content;
}
 

public Blob getContent() {
 return content;
}
 

public Blob getImage() {
	return image;
}
 

public void setImage(Blob image) {
	this.image = image;
}
 

private void customTask( final HttpServletRequest request, final PersistenceManager pm ) throws IOException {
	LOGGER.fine( "Key: " + request.getParameter( "key" ) + ", file type: " + request.getParameter( "fileType" ) );
	
	final FileType fileType = FileType.fromString( request.getParameter( "fileType" ) );
	if ( fileType == null ) {
		LOGGER.severe( "Invalid File type!" );
		return;
	}
	
	final Key key = KeyFactory.stringToKey( request.getParameter( "key" ) );
	
	final DatastoreService ds = DatastoreServiceFactory.getDatastoreService();
       final Entity e;
	try {
        e = ds.get( key );
       } catch ( final EntityNotFoundException enfe ) {
		LOGGER.log( Level.WARNING, "Entity not found!", enfe );
        return;
       }
	
       if ( !e.getKind().equals( "Rep" ) && !e.getKind().equals( "Smpd" ) && !e.getKind().equals( "OtherFile" ) ) {
		LOGGER.severe( "Invalid Entity kind:" + e.getKind() );
		return;
       }
       
       if ( (Long) e.getProperty( "v" ) == 4 ) {
		LOGGER.warning( "Entity is already v4!" );
		return;
	}
       
       // Update common properties:
       // TODO
       final int size = ( (Long) e.getProperty( "size" ) ).intValue();
       if ( size < FileServlet.DATASTORE_CONTENT_STORE_LIMIT ) {
           final FileService fileService = FileServiceFactory.getFileService();
   		final AppEngineFile   appeFile = new AppEngineFile( FileSystem.BLOBSTORE, ( (BlobKey) e.getProperty( "blobKey" ) ).getKeyString() );
   		final FileReadChannel channel  = fileService.openReadChannel( appeFile, false );
   		final byte[]          content  = new byte[ size ];
   		final ByteBuffer      wrapper  = ByteBuffer.wrap( content );
   		while ( channel.read( wrapper ) > 0 )
   			;
   		channel.close();
   		
   		e.setProperty( "content", new Blob( content ) );
   		e.setProperty( "blobKey", null );
   		fileService.delete( appeFile );
       }
       e.setUnindexedProperty( "blobKey", e.getProperty( "blobKey" ) );
       e.setUnindexedProperty( "content", e.getProperty( "content" ) );
       
       switch ( fileType ) {
       case SC2REPLAY :
           e.setUnindexedProperty( "matchup", e.getProperty( "matchup" ) );
       	break;
       case MOUSE_PRINT :
       	break;
       case OTHER :
       	break;
       default:
       	throw new RuntimeException( "Invalid file type!" );
       }
       
       // UPGRADE COMPLETE!
	e.setProperty( "v", 4 );
	ds.put( e );
}
 

/** Deserializes the specified object from a Blob using an {@link ObjectInputStream}. */
private V deserialize(Entity entity) throws IOException {
  Blob blob = (Blob) entity.getProperty(FIELD_VALUE);
  return IOUtils.deserialize(blob.getBytes());
}
 

public ShardedValue(Entity entity) {
  super(entity);
  this.shardId = (Long) entity.getProperty(SHARD_ID_PROPERTY);
  this.value = ((Blob) entity.getProperty(VALUE_PROPERTY)).getBytes();
}
 

public FanoutTaskRecord(Entity entity) {
  super(entity);
  Blob payloadBlob = (Blob) entity.getProperty(PAYLOAD_PROPERTY);
  payload = payloadBlob.getBytes();
}
 

@Override
public Entity toEntity() {
  Entity entity = toProtoEntity();
  entity.setUnindexedProperty(PAYLOAD_PROPERTY, new Blob(payload));
  return entity;
}
 

@Override
public void finishObjectCreation(RawGcsCreationToken token, ByteBuffer chunk, long timeoutMillis)
    throws IOException {
  ensureInitialized();
  Token t = append(token, chunk);

  int totalBytes = 0;

  BlobKey blobKey = getBlobKeyForFilename(t.filename);
  try (WritableByteChannel outputChannel = Channels.newChannel(blobStorage.storeBlob(blobKey))) {
    for (ByteBuffer buffer : inMemoryData.get(t.filename)){
      totalBytes += buffer.remaining();
      outputChannel.write(buffer);
    }
    inMemoryData.remove(t.filename);
  }

  String mimeType = t.options.getMimeType();
  if (Strings.isNullOrEmpty(mimeType)) {
    mimeType = "application/octet-stream";
  }

  BlobInfo blobInfo = new BlobInfo(
      blobKey, mimeType, new Date(), getPathForGcsFilename(t.filename),
      totalBytes);

  String namespace = NamespaceManager.get();
  try {
    NamespaceManager.set("");
    String blobKeyString = blobInfo.getBlobKey().getKeyString();
    Entity blobInfoEntity =
        new Entity(GOOGLE_STORAGE_FILE_KIND, blobKeyString);
    blobInfoEntity.setProperty(BlobInfoFactory.CONTENT_TYPE, blobInfo.getContentType());
    blobInfoEntity.setProperty(BlobInfoFactory.CREATION, blobInfo.getCreation());
    blobInfoEntity.setProperty(BlobInfoFactory.FILENAME, blobInfo.getFilename());
    blobInfoEntity.setProperty(BlobInfoFactory.SIZE, blobInfo.getSize());
    datastore.put(blobInfoEntity);
  } finally {
    NamespaceManager.set(namespace);
  }

  Entity e = new Entity(makeKey(t.filename));
  ByteArrayOutputStream bout = new ByteArrayOutputStream();
  try (ObjectOutputStream oout = new ObjectOutputStream(bout)) {
    oout.writeObject(t.options);
  }
  e.setUnindexedProperty(OPTIONS_PROP, new Blob(bout.toByteArray()));
  e.setUnindexedProperty(CREATION_TIME_PROP, System.currentTimeMillis());
  e.setUnindexedProperty(FILE_LENGTH_PROP, totalBytes);
  datastore.put(null, e);
}
 

/**
 * Set the value of metaSerial
 *
 * @param newmetaSerial new value of metaSerial
 */
public void setMetaSerial(Blob newmetaSerial) {
    this.metaSerial = newmetaSerial;
}
 

/**
 * Get the value of metaSerial
 *
 * @return the value of metaSerial
 */
public Blob getMetaSerial() {
    return this.metaSerial;
}
 

/**
 * Set the value of boxesSerial
 *
 * @param newboxesSerial new value of boxesSerial
 */
public void setPuzzleSerial(Blob newboxesSerial) {
    this.boxesSerial = newboxesSerial;
}
 

/**
 * Get the value of boxesSerial
 *
 * @return the value of boxesSerial
 */
public Blob getPuzzleSerial() {
    return this.boxesSerial;
}
 

/**
 * Set the value of metaSerial
 *
 * @param newmetaSerial new value of metaSerial
 */
public void setMetaSerial(Blob newmetaSerial) {
    this.metaSerial = newmetaSerial;
}