From ed97e22d575a8f3a495484354257c92ce40c9a17 Mon Sep 17 00:00:00 2001 From: Stephan Hohe Date: Fri, 19 Apr 2024 01:52:06 +0200 Subject: [PATCH 1/5] Test input stream with multiple chunks --- parser_test.ts | 76 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/parser_test.ts b/parser_test.ts index 0ae4b48..593ecec 100644 --- a/parser_test.ts +++ b/parser_test.ts @@ -20,6 +20,52 @@ import { PullResult, } from './parser.ts'; +function byteChunkStream(bytes: Uint8Array): ReadableStream { + return new ReadableStream({ + start(controller: ReadableStreamDefaultController) { + for (let pos = 0; pos < bytes.length; ++pos) { + controller.enqueue(bytes.subarray(pos, pos+1)); + } + controller.close(); + } + }); +} + +function charChunkStream(str: string): ReadableStream { + return new ReadableStream({ + start(controller: ReadableStreamDefaultController) { + for (let pos = 0; pos < str.length; ++pos) { + controller.enqueue(str.substring(pos, pos+1)); + } + controller.close(); + } + }); +} + +Deno.test('byteChunkStream writes single byte chunks', async () => { + const input = (new TextEncoder()).encode("abc"); + const stream = byteChunkStream(input); + const output = new Uint8Array(input.length); + let outputpos = 0; + for await (const chunk of stream) { + assertEquals(chunk.length, 1); + output.set(chunk, outputpos); + outputpos += chunk.length; + } + assertEquals(output, input); +}); + +Deno.test('charChunkStream writes single char chunks', async () => { + const input = "abc"; + const stream = charChunkStream(input); + let output = ""; + for await (const chunk of stream) { + assertEquals(chunk.length, 1); + output = output + chunk; + } + assertEquals(output, input); +}); + Deno.test('ParserBase chunk & hasNext & readNext & position', () => { // protected -> public visiblity class TestParser extends ParserBase { @@ -74,6 +120,36 @@ Deno.test('SAXParser on & parse(Deno.Reader)', async () => { assertEquals(elementCount, 18); }); +Deno.test('SAXParser UnderlyingSink chunks', async () => { + const parser = new SAXParser(); + + const input = (new TextEncoder()).encode("ä"); + parser.on('text', (text) => { + assertEquals(text, "\u00E4"); + }); + await byteChunkStream(input).pipeTo(new WritableStream(parser)); +}); + +Deno.test('SAXParser parse(ReadableStream)', async () => { + const parser = new SAXParser(); + parser.on('text', (text) => { + assertEquals(text, "\u00E4"); + }); + + const input = (new TextEncoder()).encode("ä"); + await parser.parse(byteChunkStream(input)); +}); + +Deno.test('SAXParser parse(ReadableStream)', async () => { + const parser = new SAXParser(); + parser.on('text', (text) => { + assertEquals(text, "\u00E4"); + }); + + const input = "ä"; + await parser.parse(charChunkStream(input)); +}); + Deno.test('SAXParser parse(Uint8Array)', () => { const parser = new SAXParser(); let flag = false; From 32a9b0cde516eee144e1cc3b6ed733840b732cea Mon Sep 17 00:00:00 2001 From: Stephan Hohe Date: Thu, 2 Apr 2026 19:30:07 +0200 Subject: [PATCH 2/5] Handle encoding when using `SAXParser` as an `UnderlyingSink` --- parser.ts | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/parser.ts b/parser.ts index f3eca17..7a63027 100644 --- a/parser.ts +++ b/parser.ts @@ -166,7 +166,12 @@ export class SAXParser extends ParserBase implements UnderlyingSink // deno-lint-ignore no-explicit-any private _listeners: { [name: string]: ((...arg: any[]) => void)[] } = {}; private _controller?: WritableStreamDefaultController; - private _encoding?: string; + private _decoder: TextDecoder; + + constructor(encoding: string = "UTF-8") { + super(); + this._decoder = new TextDecoder(encoding); + } protected fireListeners(event: XMLParseEvent) { const [name, ...args] = event; @@ -208,7 +213,18 @@ export class SAXParser extends ParserBase implements UnderlyingSink try { this._controller = controller; // TextDecoder can resolve BOM. - this.chunk = new TextDecoder(this._encoding).decode(chunk); + this.chunk = this._decoder.decode(chunk, {stream: true}); + this.run(); + } finally { + this._controller = undefined; + } + } + + close(controller?: WritableStreamDefaultController) { + try { + this._controller = controller; + // Process any remaining data still pending in `_decoder` + this.chunk = this._decoder.decode(new Uint8Array(), {stream: false}); this.run(); } finally { this._controller = undefined; @@ -220,18 +236,19 @@ export class SAXParser extends ParserBase implements UnderlyingSink * @param source Target XML. * @param encoding When the source is Deno.Reader or Uint8Array, specify the encoding. */ - async parse(source: ReadableStream | Uint8Array | string, encoding?: string) { - this._encoding = encoding; + async parse(source: ReadableStream | Uint8Array | string, encoding?: string /** @deprecated Use constructor parameter instead */) { + if (encoding !== undefined) { + this._decoder = new TextDecoder(encoding); + } if (typeof source === 'string') { this.chunk = source; this.run(); } else if (source instanceof Uint8Array) { this.write(source); + this.close(); } else { - await source.pipeThrough( - new TextDecoderStream(this._encoding) - ).pipeTo( - new WritableStream({ write: str => this.parse(str, encoding) }), + await source.pipeTo( + new WritableStream(this), ); } } From 594b3dc74a8af963c821cb73c8173165da443194 Mon Sep 17 00:00:00 2001 From: Stephan Hohe Date: Fri, 3 Apr 2026 20:08:41 +0200 Subject: [PATCH 3/5] Not all types are processed --- parser.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parser.ts b/parser.ts index 7a63027..4dd7f8e 100644 --- a/parser.ts +++ b/parser.ts @@ -236,7 +236,7 @@ export class SAXParser extends ParserBase implements UnderlyingSink * @param source Target XML. * @param encoding When the source is Deno.Reader or Uint8Array, specify the encoding. */ - async parse(source: ReadableStream | Uint8Array | string, encoding?: string /** @deprecated Use constructor parameter instead */) { + async parse(source: ReadableStream | Uint8Array | string, encoding?: string /** @deprecated Use constructor parameter instead */) { if (encoding !== undefined) { this._decoder = new TextDecoder(encoding); } From b80c89f7f57d5dde4eb86cb358917a812c09df21 Mon Sep 17 00:00:00 2001 From: Stephan Hohe Date: Sat, 4 Apr 2026 02:14:41 +0200 Subject: [PATCH 4/5] Add parsing of `ReadableStream` --- parser.ts | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/parser.ts b/parser.ts index 4dd7f8e..1aea8c4 100644 --- a/parser.ts +++ b/parser.ts @@ -162,7 +162,7 @@ export interface SAXEvent { /** * SAX-style XML parser. */ -export class SAXParser extends ParserBase implements UnderlyingSink { +export class SAXParser extends ParserBase implements UnderlyingSink, UnderlyingSink { // deno-lint-ignore no-explicit-any private _listeners: { [name: string]: ((...arg: any[]) => void)[] } = {}; private _controller?: WritableStreamDefaultController; @@ -209,11 +209,16 @@ export class SAXParser extends ParserBase implements UnderlyingSink * @param chunk XML data chunk * @param controller error reporter, Deno writable stream uses internal. */ - write(chunk: Uint8Array, controller?: WritableStreamDefaultController) { + write(chunk: Uint8Array|string, controller?: WritableStreamDefaultController) { try { this._controller = controller; // TextDecoder can resolve BOM. - this.chunk = this._decoder.decode(chunk, {stream: true}); + if (typeof chunk === 'string') { + this.chunk = chunk; + } + else { + this.chunk = this._decoder.decode(chunk, {stream: true}); + } this.run(); } finally { this._controller = undefined; @@ -224,6 +229,8 @@ export class SAXParser extends ParserBase implements UnderlyingSink try { this._controller = controller; // Process any remaining data still pending in `_decoder` + // Even if `_decoder` was unused because this parser processed strings, + // processing this empty chunk doesn't hurt this.chunk = this._decoder.decode(new Uint8Array(), {stream: false}); this.run(); } finally { @@ -236,7 +243,7 @@ export class SAXParser extends ParserBase implements UnderlyingSink * @param source Target XML. * @param encoding When the source is Deno.Reader or Uint8Array, specify the encoding. */ - async parse(source: ReadableStream | Uint8Array | string, encoding?: string /** @deprecated Use constructor parameter instead */) { + async parse(source: ReadableStream | ReadableStream | Uint8Array | string, encoding?: string /** @deprecated Use constructor parameter instead */) { if (encoding !== undefined) { this._decoder = new TextDecoder(encoding); } @@ -247,8 +254,10 @@ export class SAXParser extends ParserBase implements UnderlyingSink this.write(source); this.close(); } else { + type ReadableStreamContent = T extends ReadableStream ? Content : never; + type SourceStreamContent = ReadableStreamContent; await source.pipeTo( - new WritableStream(this), + new WritableStream(this), ); } } From dada49f132c7877de37a920e51fd7bab67c7507b Mon Sep 17 00:00:00 2001 From: Stephan Hohe Date: Sat, 4 Apr 2026 15:03:01 +0200 Subject: [PATCH 5/5] Polishing --- parser.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/parser.ts b/parser.ts index 1aea8c4..be46d17 100644 --- a/parser.ts +++ b/parser.ts @@ -212,11 +212,11 @@ export class SAXParser extends ParserBase implements UnderlyingSink, write(chunk: Uint8Array|string, controller?: WritableStreamDefaultController) { try { this._controller = controller; - // TextDecoder can resolve BOM. if (typeof chunk === 'string') { this.chunk = chunk; } else { + // TextDecoder can resolve BOM. this.chunk = this._decoder.decode(chunk, {stream: true}); } this.run(); @@ -230,7 +230,7 @@ export class SAXParser extends ParserBase implements UnderlyingSink, this._controller = controller; // Process any remaining data still pending in `_decoder` // Even if `_decoder` was unused because this parser processed strings, - // processing this empty chunk doesn't hurt + // processing this empty chunk doesn't hurt. this.chunk = this._decoder.decode(new Uint8Array(), {stream: false}); this.run(); } finally {