Skip to content

Commit eac3c35

Browse files
committed
PR comments: Update incremental encoder to be for single chunks and use nester arrays
1 parent aa89a50 commit eac3c35

File tree

6 files changed

+108
-96
lines changed

6 files changed

+108
-96
lines changed

packages/dds/tree/src/feature-libraries/chunked-forest/codec/chunkDecoding.ts

Lines changed: 15 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ import {
4141
type EncodedAnyShape,
4242
type EncodedChunkShape,
4343
type EncodedFieldBatch,
44-
type EncodedIncrementalShape,
44+
type EncodedIncrementalChunkShape,
4545
type EncodedInlineArray,
4646
type EncodedNestedArray,
4747
type EncodedTreeShape,
@@ -90,8 +90,8 @@ const decoderLibrary = new DiscriminatedUnionDispatcher<
9090
d(shape: EncodedAnyShape): ChunkDecoder {
9191
return anyDecoder;
9292
},
93-
e(shape: EncodedIncrementalShape, cache): ChunkDecoder {
94-
return new IncrementalFieldDecoder(cache);
93+
e(shape: EncodedIncrementalChunkShape, cache): ChunkDecoder {
94+
return new IncrementalChunkDecoder(cache);
9595
},
9696
});
9797

@@ -235,28 +235,24 @@ export class InlineArrayDecoder implements ChunkDecoder {
235235
}
236236

237237
/**
238-
* Decoder for {@link EncodedIncrementalShape}s.
238+
* Decoder for {@link EncodedIncrementalChunkShape}s.
239239
*/
240-
export class IncrementalFieldDecoder implements ChunkDecoder {
240+
export class IncrementalChunkDecoder implements ChunkDecoder {
241241
public constructor(private readonly cache: DecoderContext<EncodedChunkShape>) {}
242242
public decode(_: readonly ChunkDecoder[], stream: StreamCursor): TreeChunk {
243-
const chunkReferenceIds = readStream(stream);
243+
const chunkReferenceId = readStream(stream);
244244
assert(
245-
Array.isArray(chunkReferenceIds),
246-
"Expected incremental field's data to be an array of chunk reference ids",
245+
typeof chunkReferenceId === "number",
246+
"Expected incremental field's data to be a chunk reference id",
247247
);
248248

249-
const chunks: TreeChunk[] = [];
250-
for (const chunkReferenceId of chunkReferenceIds) {
251-
assert(typeof chunkReferenceId === "number", "unexpected chunk reference id");
252-
assert(
253-
this.cache.incrementalDecoder !== undefined,
254-
"incremental decoder not available for incremental field decoding",
255-
);
256-
const batch = this.cache.incrementalDecoder.getEncodedIncrementalChunk(chunkReferenceId);
257-
assert(batch !== undefined, "Incremental chunk data missing");
258-
chunks.push(...genericDecode(decoderLibrary, this.cache, batch, anyDecoder));
259-
}
249+
assert(
250+
this.cache.incrementalDecoder !== undefined,
251+
"incremental decoder not available for incremental field decoding",
252+
);
253+
const batch = this.cache.incrementalDecoder.getEncodedIncrementalChunk(chunkReferenceId);
254+
assert(batch !== undefined, "Incremental chunk data missing");
255+
const chunks = genericDecode(decoderLibrary, this.cache, batch, anyDecoder);
260256
return aggregateChunks(chunks);
261257
}
262258
}

packages/dds/tree/src/feature-libraries/chunked-forest/codec/compressedEncode.ts

Lines changed: 73 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -346,14 +346,45 @@ export class InlineArrayShape
346346
}
347347
}
348348

349+
/**
350+
* Encodes the shape for a nested array as {@link EncodedNestedArray} shape.
351+
*/
352+
class NestedShape extends ShapeGeneric<EncodedChunkShape> {
353+
/**
354+
* @param innerShape - the shape of each item in this nested array.
355+
*/
356+
public constructor(public readonly innerShape: Shape) {
357+
super();
358+
}
359+
360+
public encodeShape(
361+
identifiers: DeduplicationTable<string>,
362+
shapes: DeduplicationTable<Shape>,
363+
): EncodedChunkShape {
364+
const shape: EncodedNestedArray =
365+
shapes.valueToIndex.get(this.innerShape) ??
366+
fail(0xb4f /* index for shape not found in table */);
367+
return {
368+
a: shape,
369+
};
370+
}
371+
372+
public countReferencedShapesAndIdentifiers(
373+
identifiers: Counter<string>,
374+
shapeDiscovered: (shape: Shape) => void,
375+
): void {
376+
shapeDiscovered(this.innerShape);
377+
}
378+
}
379+
349380
/**
350381
* Encodes a field as a nested array with the {@link EncodedNestedArray} shape.
351382
*/
352-
export class NestedArrayShape extends ShapeGeneric<EncodedChunkShape> implements FieldEncoder {
383+
export class NestedArrayShape extends NestedShape implements FieldEncoder {
353384
public readonly shape: Shape;
354385

355386
public constructor(public readonly inner: NodeEncoder) {
356-
super();
387+
super(inner.shape);
357388
this.shape = this;
358389
}
359390

@@ -382,83 +413,39 @@ export class NestedArrayShape extends ShapeGeneric<EncodedChunkShape> implements
382413
outputBuffer.push(buffer);
383414
}
384415
}
385-
386-
public encodeShape(
387-
identifiers: DeduplicationTable<string>,
388-
shapes: DeduplicationTable<Shape>,
389-
): EncodedChunkShape {
390-
const shape: EncodedNestedArray =
391-
shapes.valueToIndex.get(this.inner.shape) ??
392-
fail(0xb4f /* index for shape not found in table */);
393-
return {
394-
a: shape,
395-
};
396-
}
397-
398-
public countReferencedShapesAndIdentifiers(
399-
identifiers: Counter<string>,
400-
shapeDiscovered: (shape: Shape) => void,
401-
): void {
402-
shapeDiscovered(this.inner.shape);
403-
}
404416
}
405417

406418
/**
407-
* Encodes a field that supports incremental encoding. The chunks in this field will be encoded separately.
408-
* The encoded data for this field will be an array of {@link ChunkReferenceId}s, one for each of its chunks.
419+
* Encodes a chunk with the {@link EncodedIncrementalChunkShape} shape.
420+
* This chunks will be encoded separately, i.e., the contents of the chunk will not be part of the main buffer.
421+
* A reference to the chunk will be stored in the main buffer as an {@link ChunkReferenceId}.
409422
*/
410-
export class IncrementalFieldShape
411-
extends ShapeGeneric<EncodedChunkShape>
412-
implements FieldEncoder
413-
{
414-
public constructor() {
415-
super();
416-
}
417-
423+
export class IncrementalChunkShape extends ShapeGeneric<EncodedChunkShape> {
418424
/**
419425
* Encodes all the nodes in the chunk at the cursor position using `InlineArrayShape`.
420426
*/
421-
private encodeNodesInChunk(chunk: TreeChunk, cache: EncoderCache): BufferFormat {
427+
public static encodeChunk(chunk: TreeChunk, cache: EncoderCache): BufferFormat {
428+
const chunkOutputBuffer: BufferFormat = [];
429+
const nodesEncoder = asNodesEncoder(anyNodeEncoder);
422430
const chunkCursor = chunk.cursor();
423431
chunkCursor.firstNode();
424-
const inlineArrayShape = new InlineArrayShape(
425-
chunkCursor.chunkLength,
426-
asNodesEncoder(anyNodeEncoder),
427-
);
428-
const chunkOutputBuffer: BufferFormat = [];
429-
inlineArrayShape.encodeNodes(chunkCursor, cache, chunkOutputBuffer);
432+
const chunkLength = chunkCursor.chunkLength;
433+
for (let index = 0; index < chunkLength; index++) {
434+
nodesEncoder.encodeNodes(chunkCursor, cache, chunkOutputBuffer);
435+
}
430436
assert(
431437
chunkCursor.mode === CursorLocationType.Fields,
432438
"should return to fields mode when finished encoding",
433439
);
434440
return chunkOutputBuffer;
435441
}
436442

437-
public encodeField(
438-
cursor: ITreeCursorSynchronous,
439-
cache: EncoderCache,
440-
outputBuffer: BufferFormat,
441-
): void {
442-
assert(
443-
cache.shouldEncodeIncrementally,
444-
"incremental encoding must be enabled to use IncrementalFieldShape",
445-
);
446-
447-
let chunkReferenceIds: ChunkReferenceId[] = [];
448-
if (cursor.getFieldLength() !== 0) {
449-
chunkReferenceIds = cache.encodeIncrementalField(cursor, (chunk: TreeChunk) =>
450-
this.encodeNodesInChunk(chunk, cache),
451-
);
452-
}
453-
outputBuffer.push(chunkReferenceIds);
454-
}
455-
456443
public encodeShape(
457444
identifiers: DeduplicationTable<string>,
458445
shapes: DeduplicationTable<Shape>,
459446
): EncodedChunkShape {
460447
return {
461-
e: 0 /* EncodedIncrementalShape */,
448+
e: 0 /* EncodedIncrementalChunkShape */,
462449
};
463450
}
464451

@@ -472,6 +459,34 @@ export class IncrementalFieldShape
472459
}
473460
}
474461

462+
/**
463+
* Encodes an incremental field whose chunks are encoded separately and referenced by their {@link ChunkReferenceId}.
464+
* The shape of the content of this field is {@link NestedShape} where the items in the array are
465+
* the {@link ChunkReferenceId}s of the encoded chunks.
466+
*/
467+
export const incrementalFieldEncoder: FieldEncoder = {
468+
encodeField(
469+
cursor: ITreeCursorSynchronous,
470+
cache: EncoderCache,
471+
outputBuffer: BufferFormat,
472+
): void {
473+
assert(
474+
cache.shouldEncodeIncrementally,
475+
"incremental encoding must be enabled to use IncrementalFieldShape",
476+
);
477+
478+
let chunkReferenceIds: ChunkReferenceId[] = [];
479+
if (cursor.getFieldLength() !== 0) {
480+
chunkReferenceIds = cache.encodeIncrementalField(cursor, (chunk: TreeChunk) =>
481+
IncrementalChunkShape.encodeChunk(chunk, cache),
482+
);
483+
}
484+
outputBuffer.push(chunkReferenceIds);
485+
},
486+
487+
shape: new NestedShape(new IncrementalChunkShape() /* innerShape */),
488+
};
489+
475490
/**
476491
* Encode `value` with `shape` into `outputBuffer`.
477492
*

packages/dds/tree/src/feature-libraries/chunked-forest/codec/format.ts

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,12 @@ export const EncodedInlineArray = Type.Object(
4747
*/
4848
export const EncodedAnyShape = Type.Literal(0);
4949

50+
/**
51+
* Encoded content is a {@link ChunkReferenceId}.
52+
* This represents the shape of a chunk that is encoded separately and is referenced by its reference id.
53+
*/
54+
export const EncodedIncrementalChunkShape = Type.Literal(0);
55+
5056
/**
5157
* Content of the encoded field is specified by the Shape referenced by the ShapeIndex.
5258
* This is a tuple for conciseness.
@@ -64,12 +70,6 @@ export const EncodedFieldShape = Type.Tuple([
6470

6571
export type EncodedFieldShape = Static<typeof EncodedFieldShape>;
6672

67-
/**
68-
* Content of the encoded field is an array of chunk reference ids, one for each chunk. Each chunk in the field is
69-
* encoded separately and is referenced by its reference id.
70-
*/
71-
export const EncodedIncrementalShape = Type.Literal(0);
72-
7373
enum CounterRelativeTo {
7474
// Relative to previous node of same type in depth first pre-order traversal.
7575
PreviousNodeOfType_DepthFirstPreOrder,
@@ -186,9 +186,9 @@ export const EncodedChunkShape = Type.Object(
186186
*/
187187
d: Type.Optional(EncodedAnyShape),
188188
/**
189-
* {@link EncodedIncrementalShape} union member.
189+
* {@link EncodedIncrementalChunkShape} union member.
190190
*/
191-
e: Type.Optional(EncodedIncrementalShape),
191+
e: Type.Optional(EncodedIncrementalChunkShape),
192192
},
193193
unionOptions,
194194
);
@@ -199,7 +199,7 @@ export type EncodedNestedArray = Static<typeof EncodedNestedArray>;
199199
export type EncodedInlineArray = Static<typeof EncodedInlineArray>;
200200
export type EncodedTreeShape = Static<typeof EncodedTreeShape>;
201201
export type EncodedAnyShape = Static<typeof EncodedAnyShape>;
202-
export type EncodedIncrementalShape = Static<typeof EncodedIncrementalShape>;
202+
export type EncodedIncrementalChunkShape = Static<typeof EncodedIncrementalChunkShape>;
203203

204204
export const EncodedFieldBatch = EncodedFieldBatchGeneric(version, EncodedChunkShape);
205205
export type EncodedFieldBatch = Static<typeof EncodedFieldBatch>;

packages/dds/tree/src/feature-libraries/chunked-forest/codec/schemaBasedEncode.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,12 +23,12 @@ import {
2323
EncoderCache,
2424
type FieldEncoder,
2525
type FieldShaper,
26-
IncrementalFieldShape,
2726
type KeyedFieldEncoder,
2827
type TreeShaper,
2928
anyNodeEncoder,
3029
asFieldEncoder,
3130
compressedEncode,
31+
incrementalFieldEncoder,
3232
} from "./compressedEncode.js";
3333
import type { FieldBatch } from "./fieldBatch.js";
3434
import { type EncodedFieldBatch, type EncodedValueShape, SpecialField } from "./format.js";
@@ -137,7 +137,7 @@ export function treeShaper(
137137
const fieldEncoder =
138138
shouldEncodeIncrementally &&
139139
(key === "notes" || key === "label" || key === "labelText")
140-
? new IncrementalFieldShape()
140+
? incrementalFieldEncoder
141141
: fieldHandler.shapeFromField(field);
142142
objectNodeFields.push({
143143
key,

packages/dds/tree/src/feature-libraries/forest-summary/forestSummarizer.ts

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ import type { Format } from "./format.js";
4646
import { ForestIncrementalSummaryBuilder } from "./incrementalSummaryBuilder.js";
4747

4848
/**
49-
* The key for the blob in the summary containing forest's contents.
49+
* The key for the blob in the summary containing the forest's contents.
5050
*/
5151
export const forestSummaryContentKey = "ForestTree";
5252

@@ -118,8 +118,8 @@ export class ForestSummarizer implements Summarizable {
118118
});
119119

120120
const forestSummaryBuilder = new SummaryTreeBuilder();
121-
// Let the incremental summary builder know that we are starting a new summary. It returns whether
122-
// incremental encoding is enabled.
121+
// Let the incremental summary builder know that we are starting a new summary.
122+
// It returns whether incremental encoding is enabled.
123123
const shouldEncodeIncrementally = this.incrementalSummaryBuilder.startingSummary(
124124
forestSummaryBuilder,
125125
fullTree,

packages/dds/tree/src/feature-libraries/forest-summary/incrementalSummaryBuilder.ts

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -48,12 +48,12 @@ interface ChunkSummaryProperties {
4848
* The reference ID of the chunk which uniquely identifies it under its parent's summary tree.
4949
* The summary for this chunk will be stored against this reference ID as key in the summary tree.
5050
*/
51-
referenceId: ChunkReferenceId;
51+
readonly referenceId: ChunkReferenceId;
5252
/**
5353
* The path for this chunk's summary in the summary tree relative to the forest's summary tree.
5454
* This path is used to generate a summary handle for the chunk if it doesn't change between summaries.
5555
*/
56-
summaryPath: string;
56+
readonly summaryPath: string;
5757
}
5858

5959
/**
@@ -74,12 +74,13 @@ interface TrackedSummaryProperties {
7474
*/
7575
readonly fullTree: boolean;
7676
/**
77-
* Represents the path of a chunk in the summary tree relative to the forest's summary tree. Each item in the
78-
* array is a reference ID of a chunk in the summary tree starting from the chunk under forest summary tree.
77+
* Represents the path of a chunk in the summary tree relative to the forest's summary tree.
78+
* Each item in the array is the {@link ChunkReferenceId} of a chunk in the summary tree starting
79+
* from the chunk under forest summary tree.
7980
* When a chunk is summarized, this array will be used to generate the path for the chunk's summary in the
8081
* summary tree.
8182
*/
82-
readonly chunkSummaryPath: number[];
83+
readonly chunkSummaryPath: ChunkReferenceId[];
8384
/**
8485
* The parent summary builder to use to build the incremental summary tree. When a chunk is being summarized,
8586
* it will add its summary to this builder against its reference ID.

0 commit comments

Comments
 (0)