本文整理汇总了Golang中github.com/janelia-flyem/dvid/storage.KeyValueBatcher类的典型用法代码示例。如果您正苦于以下问题:Golang KeyValueBatcher类的具体用法?Golang KeyValueBatcher怎么用?Golang KeyValueBatcher使用的例子?那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。
在下文中一共展示了KeyValueBatcher类的9个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于我们的系统推荐出更棒的Golang代码示例。
示例1: deleteBlock
func (d *Data) deleteBlock(ctx *datastore.VersionedCtx, block labels.DeleteBlock, batcher storage.KeyValueBatcher) {
batch := batcher.NewBatch(ctx)
// Iterate through this block of labels and get set of labels.
blockBytes := len(block.Data)
if blockBytes != int(d.BlockSize.Prod())*8 {
dvid.Criticalf("Deserialized label block %d bytes, not uint64 size times %d block elements\n",
blockBytes, d.BlockSize.Prod())
return
}
labelSet := make(map[uint64]struct{})
for i := 0; i < blockBytes; i += 8 {
label := binary.LittleEndian.Uint64(block.Data[i : i+8])
if label != 0 {
labelSet[label] = struct{}{}
}
}
// Go through all non-zero labels and delete the corresponding labelvol k/v pair.
zyx := block.Index.ToIZYXString()
for label := range labelSet {
tk := NewTKey(label, zyx)
batch.Delete(tk)
}
if err := batch.Commit(); err != nil {
dvid.Criticalf("Bad sync in labelvol. Couldn't commit block %s\n", zyx.Print())
}
return
}
开发者ID:jwohlwend,项目名称:dvid,代码行数:30,代码来源:sync.go
示例2: mergeLabels
func (d *Data) mergeLabels(batcher storage.KeyValueBatcher, v dvid.VersionID, op labels.MergeOp) error {
d.Lock()
defer d.Unlock()
d.StartUpdate()
ctx := datastore.NewVersionedCtx(d, v)
batch := batcher.NewBatch(ctx)
// Get the target label
targetTk := NewLabelTKey(op.Target)
targetElems, err := getElements(ctx, targetTk)
if err != nil {
return fmt.Errorf("get annotations for instance %q, target %d, in syncMerge: %v\n", d.DataName(), op.Target, err)
}
// Iterate through each merged label, read old elements, delete that k/v, then add it to the current target elements.
var delta DeltaModifyElements
elemsAdded := 0
for label := range op.Merged {
tk := NewLabelTKey(label)
elems, err := getElements(ctx, tk)
if err != nil {
return fmt.Errorf("unable to get annotation elements for instance %q, label %d in syncMerge: %v\n", d.DataName(), label, err)
}
if elems == nil || len(elems) == 0 {
continue
}
batch.Delete(tk)
elemsAdded += len(elems)
targetElems = append(targetElems, elems...)
// for labelsz. TODO, only do this computation if really subscribed.
for _, elem := range elems {
delta.Add = append(delta.Add, ElementPos{Label: op.Target, Kind: elem.Kind, Pos: elem.Pos})
delta.Del = append(delta.Del, ElementPos{Label: label, Kind: elem.Kind, Pos: elem.Pos})
}
}
if elemsAdded > 0 {
val, err := json.Marshal(targetElems)
if err != nil {
return fmt.Errorf("couldn't serialize annotation elements in instance %q: %v\n", d.DataName(), err)
}
batch.Put(targetTk, val)
if err := batch.Commit(); err != nil {
return fmt.Errorf("unable to commit merge for instance %q: %v\n", d.DataName(), err)
}
}
d.StopUpdate()
// Notify any subscribers of label annotation changes.
evt := datastore.SyncEvent{Data: d.DataUUID(), Event: ModifyElementsEvent}
msg := datastore.SyncMessage{Event: ModifyElementsEvent, Version: ctx.VersionID(), Delta: delta}
if err := datastore.NotifySubscribers(evt, msg); err != nil {
dvid.Criticalf("unable to notify subscribers of event %s: %v\n", evt, err)
}
return nil
}
开发者ID:tartavull,项目名称:dvid,代码行数:57,代码来源:sync.go
示例3: ingestBlock
// Note that this does not delete any removed labels in the block since we only get the CURRENT block
// and not PAST blocks. To allow mutation of label blocks, not just ingestion, we need another function.
func (d *Data) ingestBlock(ctx *datastore.VersionedCtx, block imageblk.Block, batcher storage.KeyValueBatcher) {
// Iterate through this block of labels and create RLEs for each label.
blockBytes := len(block.Data)
if blockBytes != int(d.BlockSize.Prod())*8 {
dvid.Criticalf("Deserialized label block %d bytes, not uint64 size times %d block elements\n",
blockBytes, d.BlockSize.Prod())
return
}
labelRLEs := make(map[uint64]dvid.RLEs, 10)
firstPt := block.Index.MinPoint(d.BlockSize)
lastPt := block.Index.MaxPoint(d.BlockSize)
var curStart dvid.Point3d
var voxelLabel, curLabel, maxLabel uint64
var z, y, x, curRun int32
start := 0
for z = firstPt.Value(2); z <= lastPt.Value(2); z++ {
for y = firstPt.Value(1); y <= lastPt.Value(1); y++ {
for x = firstPt.Value(0); x <= lastPt.Value(0); x++ {
voxelLabel = binary.LittleEndian.Uint64(block.Data[start : start+8])
if maxLabel < voxelLabel {
maxLabel = voxelLabel
}
start += 8
// If we hit background or have switched label, save old run and start new one.
if voxelLabel == 0 || voxelLabel != curLabel {
// Save old run
if curRun > 0 {
labelRLEs[curLabel] = append(labelRLEs[curLabel], dvid.NewRLE(curStart, curRun))
}
// Start new one if not zero label.
if voxelLabel != 0 {
curStart = dvid.Point3d{x, y, z}
curRun = 1
} else {
curRun = 0
}
curLabel = voxelLabel
} else {
curRun++
}
}
// Force break of any runs when we finish x scan.
if curRun > 0 {
labelRLEs[curLabel] = append(labelRLEs[curLabel], dvid.NewRLE(curStart, curRun))
curLabel = 0
curRun = 0
}
}
}
// Store the RLEs for each label in this block.
if maxLabel > 0 {
batch := batcher.NewBatch(ctx)
blockStr := block.Index.ToIZYXString()
for label, rles := range labelRLEs {
tk := NewTKey(label, blockStr)
rleBytes, err := rles.MarshalBinary()
if err != nil {
dvid.Errorf("Bad encoding labelvol keys for label %d: %v\n", label, err)
return
}
batch.Put(tk, rleBytes)
}
// compare-and-set MaxLabel and batch commit
d.casMaxLabel(batch, ctx.VersionID(), maxLabel)
}
}
开发者ID:jwohlwend,项目名称:dvid,代码行数:72,代码来源:sync.go
示例4: splitLabelsFine
func (d *Data) splitLabelsFine(batcher storage.KeyValueBatcher, v dvid.VersionID, op labels.DeltaSplit) error {
d.Lock()
defer d.Unlock()
d.StartUpdate()
defer d.StopUpdate()
ctx := datastore.NewVersionedCtx(d, v)
batch := batcher.NewBatch(ctx)
var delta DeltaModifyElements
toAdd := Elements{}
toDel := make(map[string]struct{})
// Iterate through each split block, get the elements, and then modify the previous and new label k/v.
for izyx, rles := range op.Split {
// Get the elements for this block.
blockPt, err := izyx.ToChunkPoint3d()
if err != nil {
return err
}
tk := NewBlockTKey(blockPt)
elems, err := getElements(ctx, tk)
if err != nil {
dvid.Errorf("getting annotations for block %s on split of %d from %d: %v\n", blockPt, op.NewLabel, op.OldLabel, err)
continue
}
// For any element within the split RLEs, add to the delete and addition lists.
for _, elem := range elems {
for _, rle := range rles {
if rle.Within(elem.Pos) {
toAdd = append(toAdd, elem)
toDel[elem.Pos.String()] = struct{}{}
// for downstream annotation syncs like labelsz. TODO: only perform if subscribed. Better: do ROI filtering here.
delta.Del = append(delta.Del, ElementPos{Label: op.OldLabel, Kind: elem.Kind, Pos: elem.Pos})
delta.Add = append(delta.Add, ElementPos{Label: op.NewLabel, Kind: elem.Kind, Pos: elem.Pos})
break
}
}
}
}
// Modify the old label k/v
if len(toDel) != 0 {
tk := NewLabelTKey(op.OldLabel)
elems, err := getElements(ctx, tk)
if err != nil {
dvid.Errorf("unable to get annotations for instance %q, old label %d in syncSplit: %v\n", d.DataName(), op.OldLabel, err)
} else {
filtered := elems[:0]
for _, elem := range elems {
if _, found := toDel[elem.Pos.String()]; !found {
filtered = append(filtered, elem)
}
}
if len(filtered) == 0 {
batch.Delete(tk)
} else {
val, err := json.Marshal(filtered)
if err != nil {
dvid.Errorf("couldn't serialize annotation elements in instance %q: %v\n", d.DataName(), err)
} else {
batch.Put(tk, val)
}
}
}
}
// Modify the new label k/v
if len(toAdd) != 0 {
tk := NewLabelTKey(op.NewLabel)
elems, err := getElements(ctx, tk)
if err != nil {
dvid.Errorf("unable to get annotations for instance %q, label %d in syncSplit: %v\n", d.DataName(), op.NewLabel, err)
} else {
elems.add(toAdd)
val, err := json.Marshal(elems)
if err != nil {
dvid.Errorf("couldn't serialize annotation elements in instance %q: %v\n", d.DataName(), err)
} else {
batch.Put(tk, val)
}
}
}
if err := batch.Commit(); err != nil {
return fmt.Errorf("bad commit in annotations %q after split: %v\n", d.DataName(), err)
}
// Notify any subscribers of label annotation changes.
evt := datastore.SyncEvent{Data: d.DataUUID(), Event: ModifyElementsEvent}
msg := datastore.SyncMessage{Event: ModifyElementsEvent, Version: ctx.VersionID(), Delta: delta}
if err := datastore.NotifySubscribers(evt, msg); err != nil {
dvid.Criticalf("unable to notify subscribers of event %s: %v\n", evt, err)
}
return nil
}
开发者ID:tartavull,项目名称:dvid,代码行数:99,代码来源:sync.go
示例5: splitLabelsCoarse
func (d *Data) splitLabelsCoarse(batcher storage.KeyValueBatcher, v dvid.VersionID, op labels.DeltaSplit) error {
d.Lock()
defer d.Unlock()
d.StartUpdate()
defer d.StopUpdate()
ctx := datastore.NewVersionedCtx(d, v)
batch := batcher.NewBatch(ctx)
// Get the elements for the old label.
oldTk := NewLabelTKey(op.OldLabel)
oldElems, err := getElements(ctx, oldTk)
if err != nil {
return fmt.Errorf("unable to get annotations for instance %q, label %d in syncSplit: %v\n", d.DataName(), op.OldLabel, err)
}
// Create a map to test each point.
splitBlocks := make(map[dvid.IZYXString]struct{})
for _, zyxStr := range op.SortedBlocks {
splitBlocks[zyxStr] = struct{}{}
}
// Move any elements that are within the split blocks.
var delta DeltaModifyElements
toDel := make(map[int]struct{})
toAdd := Elements{}
blockSize := d.blockSize()
for i, elem := range oldElems {
zyxStr := elem.Pos.ToBlockIZYXString(blockSize)
if _, found := splitBlocks[zyxStr]; found {
toDel[i] = struct{}{}
toAdd = append(toAdd, elem)
// for downstream annotation syncs like labelsz. TODO: only perform if subscribed. Better: do ROI filtering here.
delta.Del = append(delta.Del, ElementPos{Label: op.OldLabel, Kind: elem.Kind, Pos: elem.Pos})
delta.Add = append(delta.Add, ElementPos{Label: op.NewLabel, Kind: elem.Kind, Pos: elem.Pos})
}
}
if len(toDel) == 0 {
return nil
}
// Store split elements into new label elements.
newTk := NewLabelTKey(op.NewLabel)
newElems, err := getElements(ctx, newTk)
if err != nil {
return fmt.Errorf("unable to get annotations for instance %q, label %d in syncSplit: %v\n", d.DataName(), op.NewLabel, err)
}
newElems.add(toAdd)
val, err := json.Marshal(newElems)
if err != nil {
return fmt.Errorf("couldn't serialize annotation elements in instance %q: %v\n", d.DataName(), err)
}
batch.Put(newTk, val)
// Delete any split from old label elements without removing the relationships.
// This filters without allocating, using fact that a slice shares the same backing array and
// capacity as the original, so storage is reused.
filtered := oldElems[:0]
for i, elem := range oldElems {
if _, found := toDel[i]; !found {
filtered = append(filtered, elem)
}
}
// Delete or store k/v depending on what remains.
if len(filtered) == 0 {
batch.Delete(oldTk)
} else {
val, err := json.Marshal(filtered)
if err != nil {
return fmt.Errorf("couldn't serialize annotation elements in instance %q: %v\n", d.DataName(), err)
}
batch.Put(oldTk, val)
}
if err := batch.Commit(); err != nil {
return fmt.Errorf("bad commit in annotations %q after split: %v\n", d.DataName(), err)
}
// Notify any subscribers of label annotation changes.
evt := datastore.SyncEvent{Data: d.DataUUID(), Event: ModifyElementsEvent}
msg := datastore.SyncMessage{Event: ModifyElementsEvent, Version: ctx.VersionID(), Delta: delta}
if err := datastore.NotifySubscribers(evt, msg); err != nil {
dvid.Criticalf("unable to notify subscribers of event %s: %v\n", evt, err)
}
return nil
}
开发者ID:tartavull,项目名称:dvid,代码行数:89,代码来源:sync.go
示例6: deleteBlock
// If a block of labels is deleted, the associated synapse elements should be changed to zero label elements.
func (d *Data) deleteBlock(ctx *datastore.VersionedCtx, block labels.DeleteBlock, batcher storage.KeyValueBatcher) {
// Get the synaptic elements for this block
chunkPt := dvid.ChunkPoint3d(*block.Index)
tk := NewBlockTKey(chunkPt)
elems, err := getElements(ctx, tk)
if err != nil {
dvid.Errorf("err getting elements for block %s: %v\n", chunkPt, err)
return
}
if len(elems) == 0 {
return
}
blockSize := d.blockSize()
batch := batcher.NewBatch(ctx)
// Compute the strides (in bytes)
bX := blockSize[0] * 8
bY := blockSize[1] * bX
// Iterate through all element positions, finding corresponding label and storing elements.
toDel := LabelPoints{}
for _, elem := range elems {
pt := elem.Pos.Point3dInChunk(blockSize)
i := pt[2]*bY + pt[1]*bX + pt[0]*8
label := binary.LittleEndian.Uint64(block.Data[i : i+8])
toDel.add(label, elem.Pos)
}
// Delete any non-zero label elements from their respective label k/v.
var delta DeltaModifyElements
for label, pts := range toDel {
tk := NewLabelTKey(label)
elems, err := getElements(ctx, tk)
if err != nil {
dvid.Errorf("err getting elements for label %d: %v\n", label, err)
return
}
save := false
for _, pt := range pts {
deleted, changed := elems.delete(pt)
if changed {
save = true
delta.Del = append(delta.Del, ElementPos{Label: label, Kind: deleted.Kind, Pos: pt})
}
}
if save {
if len(elems) == 0 {
batch.Delete(tk)
} else {
val, err := json.Marshal(elems)
if err != nil {
dvid.Errorf("couldn't serialize annotation elements in instance %q: %v\n", d.DataName(), err)
return
}
batch.Put(tk, val)
}
}
}
if err := batch.Commit(); err != nil {
dvid.Criticalf("bad commit in annotations %q after delete block: %v\n", d.DataName(), err)
return
}
// Notify any subscribers of label annotation changes.
evt := datastore.SyncEvent{Data: d.DataUUID(), Event: ModifyElementsEvent}
msg := datastore.SyncMessage{Event: ModifyElementsEvent, Version: ctx.VersionID(), Delta: delta}
if err := datastore.NotifySubscribers(evt, msg); err != nil {
dvid.Criticalf("unable to notify subscribers of event %s: %v\n", evt, err)
}
}
开发者ID:tartavull,项目名称:dvid,代码行数:72,代码来源:sync.go
示例7: mutateBlock
// If a block of labels is mutated, adjust any label that was either removed or added.
func (d *Data) mutateBlock(ctx *datastore.VersionedCtx, block imageblk.MutatedBlock, batcher storage.KeyValueBatcher) {
// Get the synaptic elements for this block
chunkPt := dvid.ChunkPoint3d(*block.Index)
tk := NewBlockTKey(chunkPt)
elems, err := getElementsNR(ctx, tk)
if err != nil {
dvid.Errorf("err getting elements for block %s: %v\n", chunkPt, err)
return
}
if len(elems) == 0 {
return
}
blockSize := d.blockSize()
batch := batcher.NewBatch(ctx)
// Compute the strides (in bytes)
bX := blockSize[0] * 8
bY := blockSize[1] * bX
// Iterate through all element positions, finding corresponding label and storing elements.
var delta DeltaModifyElements
labels := make(map[uint64]struct{})
toAdd := LabelElements{}
toDel := LabelPoints{}
for _, elem := range elems {
pt := elem.Pos.Point3dInChunk(blockSize)
i := pt[2]*bY + pt[1]*bX + pt[0]*8
label := binary.LittleEndian.Uint64(block.Data[i : i+8])
var prev uint64
if len(block.Prev) != 0 {
prev = binary.LittleEndian.Uint64(block.Prev[i : i+8])
}
if label == prev {
continue
}
if label != 0 {
toAdd.add(label, elem)
labels[label] = struct{}{}
delta.Add = append(delta.Add, ElementPos{Label: label, Kind: elem.Kind, Pos: elem.Pos})
}
if prev != 0 {
toDel.add(prev, elem.Pos)
labels[prev] = struct{}{}
delta.Del = append(delta.Del, ElementPos{Label: prev, Kind: elem.Kind, Pos: elem.Pos})
}
}
// Modify any modified label k/v.
for label := range labels {
tk := NewLabelTKey(label)
elems, err := getElementsNR(ctx, tk)
if err != nil {
dvid.Errorf("err getting elements for label %d: %v\n", label, err)
return
}
additions, found := toAdd[label]
if found {
elems.add(additions)
}
deletions, found := toDel[label]
if found {
for _, pt := range deletions {
elems.delete(pt)
}
}
val, err := json.Marshal(elems)
if err != nil {
dvid.Errorf("couldn't serialize annotation elements in instance %q: %v\n", d.DataName(), err)
return
}
batch.Put(tk, val)
}
if err := batch.Commit(); err != nil {
dvid.Criticalf("bad commit in annotations %q after delete block: %v\n", d.DataName(), err)
return
}
// Notify any subscribers of label annotation changes.
evt := datastore.SyncEvent{Data: d.DataUUID(), Event: ModifyElementsEvent}
msg := datastore.SyncMessage{Event: ModifyElementsEvent, Version: ctx.VersionID(), Delta: delta}
if err := datastore.NotifySubscribers(evt, msg); err != nil {
dvid.Criticalf("unable to notify subscribers of event %s: %v\n", evt, err)
}
}
开发者ID:tartavull,项目名称:dvid,代码行数:85,代码来源:sync.go
示例8: ingestBlock
// If a block of labels is ingested, adjust each label's synaptic element list.
func (d *Data) ingestBlock(ctx *datastore.VersionedCtx, block imageblk.Block, batcher storage.KeyValueBatcher) {
// Get the synaptic elements for this block
chunkPt := dvid.ChunkPoint3d(*block.Index)
tk := NewBlockTKey(chunkPt)
elems, err := getElementsNR(ctx, tk)
if err != nil {
dvid.Errorf("err getting elements for block %s: %v\n", chunkPt, err)
return
}
if len(elems) == 0 {
return
}
blockSize := d.blockSize()
batch := batcher.NewBatch(ctx)
// Compute the strides (in bytes)
bX := blockSize[0] * 8
bY := blockSize[1] * bX
// Iterate through all element positions, finding corresponding label and storing elements.
added := 0
toAdd := LabelElements{}
for _, elem := range elems {
pt := elem.Pos.Point3dInChunk(blockSize)
i := (pt[2]*bY+pt[1])*bX + pt[0]*8
label := binary.LittleEndian.Uint64(block.Data[i : i+8])
if label != 0 {
toAdd.add(label, elem)
added++
}
}
// Add any non-zero label elements to their respective label k/v.
var delta DeltaModifyElements
delta.Add = make([]ElementPos, added)
i := 0
for label, addElems := range toAdd {
tk := NewLabelTKey(label)
elems, err := getElementsNR(ctx, tk)
if err != nil {
dvid.Errorf("err getting elements for label %d: %v\n", label, err)
return
}
elems.add(addElems)
val, err := json.Marshal(elems)
if err != nil {
dvid.Errorf("couldn't serialize annotation elements in instance %q: %v\n", d.DataName(), err)
return
}
batch.Put(tk, val)
for _, addElem := range addElems {
delta.Add[i] = ElementPos{Label: label, Kind: addElem.Kind, Pos: addElem.Pos}
i++
}
}
if err := batch.Commit(); err != nil {
dvid.Criticalf("bad commit in annotations %q after delete block: %v\n", d.DataName(), err)
return
}
// Notify any subscribers of label annotation changes.
evt := datastore.SyncEvent{Data: d.DataUUID(), Event: ModifyElementsEvent}
msg := datastore.SyncMessage{Event: ModifyElementsEvent, Version: ctx.VersionID(), Delta: delta}
if err := datastore.NotifySubscribers(evt, msg); err != nil {
dvid.Criticalf("unable to notify subscribers of event %s: %v\n", evt, err)
}
}
开发者ID:tartavull,项目名称:dvid,代码行数:70,代码来源:sync.go
示例9: modifyElements
func (d *Data) modifyElements(ctx *datastore.VersionedCtx, delta annotation.DeltaModifyElements, batcher storage.KeyValueBatcher) {
mods := make(map[indexedLabel]int32)
for _, elemPos := range delta.Add {
if d.inROI(elemPos) {
i := toIndexedLabel(elemPos)
mods[i]++
if elemPos.Kind.IsSynaptic() {
i = newIndexedLabel(AllSyn, elemPos.Label)
mods[i]++
}
}
}
for _, elemPos := range delta.Del {
if d.inROI(elemPos) {
i := toIndexedLabel(elemPos)
mods[i]--
if elemPos.Kind.IsSynaptic() {
i = newIndexedLabel(AllSyn, elemPos.Label)
mods[i]--
}
}
}
d.Lock()
defer d.Unlock()
// Get old counts for the modified labels.
counts, err := d.getCounts(ctx, mods)
if err != nil {
dvid.Errorf("couldn't get counts for modified labels: %v\n", err)
return
}
// Modify the keys based on the change in counts, then delete or store.
batch := batcher.NewBatch(ctx)
for il, change := range mods {
if change == 0 {
continue
}
i, label, err := decodeIndexedLabel(il)
if err != nil {
dvid.Criticalf("couldn't decode indexedLabel %s for modify elements sync of %s: %v\n", il, d.DataName(), err)
continue
}
// check if we had prior key that needs to be deleted.
count, found := counts[il]
if found {
batch.Delete(NewTypeSizeLabelTKey(i, count, label))
}
// add new count
if change < 0 && -change > int32(count) {
dvid.Criticalf("received element mod that would subtract %d with only count %d! Setting floor at 0.\n", -change, count)
change = int32(-count)
}
newcount := uint32(int32(count) + change)
// If it's at zero, we've merged or removed it so delete the count.
if newcount == 0 {
batch.Delete(NewTypeLabelTKey(i, label))
batch.Delete(NewTypeSizeLabelTKey(i, newcount, label))
continue
}
// store the data.
buf := make([]byte, 4)
binary.LittleEndian.PutUint32(buf, newcount)
batch.Put(NewTypeLabelTKey(i, label), buf)
batch.Put(NewTypeSizeLabelTKey(i, newcount, label), nil)
}
if err := batch.Commit(); err != nil {
dvid.Criticalf("bad commit in labelsz %q during sync of modify elements: %v\n", d.DataName(), err)
return
}
}
开发者ID:tartavull,项目名称:dvid,代码行数:77,代码来源:sync.go
注:本文中的github.com/janelia-flyem/dvid/storage.KeyValueBatcher类示例整理自Github/MSDocs等源码及文档管理平台,相关代码片段筛选自各路编程大神贡献的开源项目,源码版权归原作者所有,传播和使用请参考对应项目的License;未经允许,请勿转载。 |
请发表评论