下面列出了com.google.common.collect.MinMaxPriorityQueue#add ( ) 实例代码,或者点击链接到github查看源代码,也可以在右侧发表评论。
public void findKBest(int k,
float threshold,
MinMaxPriorityQueue<Pair<Float, Integer>> heap,
Vector hidden,
Vector output) {
computeOutputSoftmax(hidden, output);
for (int i = 0; i < osz; i++) {
if (output.data[i] < threshold) continue;
if (heap.size() == k && stdLog(output.data[i]) < heap.peekFirst().first()) {
continue;
}
heap.add(new Pair<>(stdLog(output.data[i]), i));
}
while (heap.size() > k) {
heap.pollLast();
}
}
public CachedPostingListCounter rebuildCache() {
MinMaxPriorityQueue<Entry> mostExpensive = MinMaxPriorityQueue
.maximumSize(32).expectedSize(32).create();
synchronized (this) {
for (ObjectLongPair<int[]> p : frequency.keyValuesView()) {
mostExpensive.add(new Entry(p.getOne(), p.getTwo()));
}
}
ObjectIntHashMap<int[]> postingListMapping = new ObjectIntHashMap<>();
int[] bitVector = new int[nDocuments];
int length = mostExpensive.size();
for (int i = 0; i < length; i++) {
Entry e = mostExpensive.removeFirst();
int[] docIds = e.docIds;
postingListMapping.put(docIds, i);
for (int docId : docIds) {
bitVector[docId] |= (1 << i);
}
}
return new CachedPostingListCounter(postingListMapping, bitVector);
}
/**
* Used when the cache is growing past its max size to clone in a single pass.
* Removes least recently used tables to get size of cache below its max size by
* the overage amount.
*/
public PTableCache cloneMinusOverage(long overage) {
assert(overage > 0);
int nToRemove = Math.max(MIN_REMOVAL_SIZE, (int)Math.ceil((currentByteSize-maxByteSize) / ((double)currentByteSize / size())) + 1);
MinMaxPriorityQueue<PTableRef> toRemove = BUILDER.expectedSize(nToRemove).create();
PTableCache newCache = new PTableCache(this.size(), this.maxByteSize, this.timeKeeper);
long toRemoveBytes = 0;
// Add to new cache, but track references to remove when done
// to bring cache at least overage amount below it's max size.
for (PTableRef tableRef : tables.values()) {
newCache.put(tableRef.table.getKey(), new PTableRef(tableRef));
toRemove.add(tableRef);
toRemoveBytes += tableRef.estSize;
if (toRemoveBytes - toRemove.peekLast().estSize > overage) {
PTableRef removedRef = toRemove.removeLast();
toRemoveBytes -= removedRef.estSize;
}
}
for (PTableRef toRemoveRef : toRemove) {
newCache.remove(toRemoveRef.table.getKey());
}
return newCache;
}
@Override
public List<LinkRelevance> getSelectedLinks() {
List<LinkRelevance> links = new ArrayList<>();
while (links.size() < numberOfLinks && !topkLinksPerDomain.isEmpty()) {
// adds the URL with max score of each domain
MinMaxPriorityQueue<LinkRelevance> topk = newPriorityQueue(numberOfLinks);
Iterator<Entry<String, MinMaxPriorityQueue<LinkRelevance>>> it = topkLinksPerDomain.entrySet().iterator();
while (it.hasNext()) {
MinMaxPriorityQueue<LinkRelevance> domain = it.next().getValue();
topk.add(domain.poll());
if (domain.isEmpty()) {
it.remove();
}
}
for(LinkRelevance link : topk) {
links.add(link);
}
}
this.topkLinksPerDomain = null; // clean-up reference
return links;
}
/**
* Used when the cache is growing past its max size to clone in a single pass.
* Removes least recently used tables to get size of cache below its max size by
* the overage amount.
*/
public PMetaDataCache cloneMinusOverage(long overage) {
assert(overage > 0);
int nToRemove = Math.max(MIN_REMOVAL_SIZE, (int)Math.ceil((currentByteSize-maxByteSize) / ((double)currentByteSize / size())) + 1);
MinMaxPriorityQueue<PTableRef> toRemove = BUILDER.expectedSize(nToRemove).create();
PMetaDataCache newCache = new PMetaDataCache(this.size(), this.maxByteSize, this.timeKeeper, this.tableRefFactory);
long toRemoveBytes = 0;
// Add to new cache, but track references to remove when done
// to bring cache at least overage amount below it's max size.
for (PTableRef tableRef : this.tables.values()) {
newCache.put(tableRef.getTable().getKey(), tableRefFactory.makePTableRef(tableRef));
toRemove.add(tableRef);
toRemoveBytes += tableRef.getEstimatedSize();
while (toRemoveBytes - toRemove.peekLast().getEstimatedSize() >= overage) {
PTableRef removedRef = toRemove.removeLast();
toRemoveBytes -= removedRef.getEstimatedSize();
}
}
for (PTableRef toRemoveRef : toRemove) {
newCache.remove(toRemoveRef.getTable().getKey());
}
return newCache;
}
@Test
public void comparatorTest() throws Exception {
MinMaxPriorityQueue<DimensionValueMetricPair> testQueue = MinMaxPriorityQueue.maximumSize(2).create();
DimensionValueMetricPair d1 = new DimensionValueMetricPair("d1", 1);
DimensionValueMetricPair d2 = new DimensionValueMetricPair("d2", 2);
DimensionValueMetricPair d3 = new DimensionValueMetricPair(30, 3);
DimensionValueMetricPair d4 = new DimensionValueMetricPair("d4", 4);
testQueue.add(d1);
testQueue.add(d2);
testQueue.add(d3);
testQueue.add(d4);
for (DimensionValueMetricPair pair : testQueue) {
Assert.assertEquals(pair.getMetricValue().intValue() > 2, true,
"Incorrect comparator for DimensionValueMetricPair, queue must retain highest metric values");
}
}
public void dfs(int k,
float threshold,
int node,
float score,
MinMaxPriorityQueue<Pair<Float, Integer>> heap,
Vector hidden) {
if (score < stdLog(threshold)) return;
if (heap.size() == k && score < heap.peekLast().first()) {
return;
}
if (tree[node].left == -1 && tree[node].right == -1) {
heap.add(new Pair<>(score, node));
if (heap.size() > k) {
Pair<Float, Integer> p = heap.pollLast();
}
return;
}
float f;
if (quant && args.getQOut()) {
f = qwo.dotRow(hidden, node - osz);
} else {
f = wo.dotRow(hidden, node - osz);
}
f = 1f / (1f + (float) Math.exp(-f));
dfs(k, threshold, tree[node].left, score + stdLog(1.0f - f), heap, hidden);
dfs(k, threshold, tree[node].right, score + stdLog(f), heap, hidden);
}
@Override
public void evaluateLink(LinkRelevance link) {
if (link.getRelevance() > minRelevance) {
String domainName = link.getTopLevelDomainName();
MinMaxPriorityQueue<LinkRelevance> domainQueue = topkLinksPerDomain.get(domainName);
if (domainQueue == null) {
domainQueue = newPriorityQueue(MAX_LINKS_PER_DOMAIN);
topkLinksPerDomain.put(domainName, domainQueue);
}
domainQueue.add(link);
}
}
@Override
protected void cleanup(Context context) throws IOException, InterruptedException {
for (String dimension : dimensionNames) {
LOGGER.info("{} records passed metric threshold for dimension {}", thresholdPassCount.get(dimension), dimension);
// Get top k
TopKDimensionToMetricsSpec topkSpec = topKDimensionToMetricsSpecMap.get(dimension);
if (topkSpec != null && topkSpec.getDimensionName() != null && topkSpec.getTopk() != null) {
// Get top k for each metric specified
Map<String, Integer> topkMetricsMap = topkSpec.getTopk();
for (Entry<String, Integer> topKEntry : topkMetricsMap.entrySet()) {
String metric = topKEntry.getKey();
int k = topKEntry.getValue();
MinMaxPriorityQueue<DimensionValueMetricPair> topKQueue = MinMaxPriorityQueue.maximumSize(k).create();
Map<Object, Number[]> dimensionToMetricsMap = dimensionNameToValuesMap.get(dimension);
for (Entry<Object, Number[]> entry : dimensionToMetricsMap.entrySet()) {
topKQueue.add(new DimensionValueMetricPair(entry.getKey(), entry.getValue()[metricToIndexMapping.get(metric)]));
}
LOGGER.info("Picking Top {} values for {} based on Metric {} : {}", k, dimension, metric, topKQueue);
for (DimensionValueMetricPair pair : topKQueue) {
topkDimensionValues.addValue(dimension, String.valueOf(pair.getDimensionValue()));
}
}
}
}
if (topkDimensionValues.getTopKDimensions().size() > 0) {
String topkValuesPath = configuration.get(TOPK_PHASE_OUTPUT_PATH.toString());
LOGGER.info("Writing top k values to {}",topkValuesPath);
FSDataOutputStream topKDimensionValuesOutputStream = fileSystem.create(
new Path(topkValuesPath + File.separator + ThirdEyeConstants.TOPK_VALUES_FILE));
OBJECT_MAPPER.writeValue((DataOutput) topKDimensionValuesOutputStream, topkDimensionValues);
topKDimensionValuesOutputStream.close();
}
}
@Test
public void givenMinMaxPriorityQueue_whenAddElementToFull_thenShouldEvictGreatestItem() {
//given
MinMaxPriorityQueue<CustomClass> queue = MinMaxPriorityQueue
.orderedBy(Comparator.comparing(CustomClass::getValue))
.maximumSize(10)
.create();
//when
IntStream
.iterate(10, i -> i - 1)
.limit(10)
.forEach(i -> queue.add(new CustomClass(i)));
//then
assertThat(queue.peekFirst().getValue()).isEqualTo(1);
assertThat(queue.peekLast().getValue()).isEqualTo(10);
//and
queue.add(new CustomClass(-1));
//then
assertThat(queue.peekFirst().getValue()).isEqualTo(-1);
assertThat(queue.peekLast().getValue()).isEqualTo(9);
//and
queue.add(new CustomClass(100));
assertThat(queue.peekFirst().getValue()).isEqualTo(-1);
assertThat(queue.peekLast().getValue()).isEqualTo(9);
}