diff --git a/package-lock.json b/package-lock.json index 617113a..c3309c2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "zon-format", - "version": "1.3.0", + "version": "1.3.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "zon-format", - "version": "1.3.0", + "version": "1.3.1", "license": "MIT", "dependencies": { "@azure/openai": "^2.0.0", diff --git a/src/__tests__/boolean-keys.test.ts b/src/__tests__/boolean-keys.test.ts new file mode 100644 index 0000000..6a63abc --- /dev/null +++ b/src/__tests__/boolean-keys.test.ts @@ -0,0 +1,127 @@ +/** + * Tests for boolean-like dictionary keys. + * + * Verifies the fix for a bug where dictionary keys like "f", "t", + * "true", "false", "null" were incorrectly parsed as boolean/null values + * instead of being preserved as strings. + * + * Fixtures wrap the boolean-like key inside a nested object or array-of-objects + * so that encode/decode exercises _formatZonNode / _parseKey — the code paths + * that contain the actual fix. + */ + +import { encode, decode } from '../index'; + +describe('Boolean-like dictionary keys', () => { + test('Key "f" should not become false', () => { + const data = { wrapper: { f: 1 } }; + const encoded = encode(data); + const decoded = decode(encoded) as any; + expect(decoded.wrapper).toEqual({ f: 1 }); + expect(Object.keys(decoded.wrapper)).toContain('f'); + expect(Object.keys(decoded.wrapper)).not.toContain('false'); + }); + + test('Key "t" should not become true', () => { + const data = { wrapper: { t: 1 } }; + const encoded = encode(data); + const decoded = decode(encoded) as any; + expect(decoded.wrapper).toEqual({ t: 1 }); + expect(Object.keys(decoded.wrapper)).toContain('t'); + expect(Object.keys(decoded.wrapper)).not.toContain('true'); + }); + + test('Key "true" should not become boolean true', () => { + const data = { wrapper: { true: 1 } }; + const encoded = encode(data); + const decoded = decode(encoded) as any; + expect(decoded.wrapper).toEqual({ true: 1 }); + expect(Object.keys(decoded.wrapper)).toContain('true'); + }); + + test('Key "false" should not become boolean false', () => { + const data = { wrapper: { false: 1 } }; + const encoded = encode(data); + const decoded = decode(encoded) as any; + expect(decoded.wrapper).toEqual({ false: 1 }); + expect(Object.keys(decoded.wrapper)).toContain('false'); + }); + + test('Key "null" should not become null', () => { + const data = { wrapper: { null: 1 } }; + const encoded = encode(data); + const decoded = decode(encoded) as any; + expect(decoded.wrapper).toEqual({ null: 1 }); + expect(Object.keys(decoded.wrapper)).toContain('null'); + }); + + test('Key "none" should not become null', () => { + const data = { wrapper: { none: 1 } }; + const encoded = encode(data); + const decoded = decode(encoded) as any; + expect(decoded.wrapper).toEqual({ none: 1 }); + expect(Object.keys(decoded.wrapper)).toContain('none'); + }); + + test('Key "nil" should not become null', () => { + const data = { wrapper: { nil: 1 } }; + const encoded = encode(data); + const decoded = decode(encoded) as any; + expect(decoded.wrapper).toEqual({ nil: 1 }); + expect(Object.keys(decoded.wrapper)).toContain('nil'); + }); + + test('Case variants of boolean-like keys are preserved', () => { + const cases: Array<[string, Record]> = [ + ['F', { F: 1 }], + ['T', { T: 1 }], + ['True', { True: 1 }], + ['False', { False: 1 }], + ['TRUE', { TRUE: 1 }], + ['FALSE', { FALSE: 1 }], + ['NULL', { NULL: 1 }], + ['NONE', { NONE: 1 }], + ['Null', { Null: 1 }], + ]; + for (const [keyName, inner] of cases) { + const data = { wrapper: inner }; + const encoded = encode(data); + const decoded = decode(encoded) as any; + expect(Object.keys(decoded.wrapper)).toContain(keyName); + expect(decoded.wrapper[keyName]).toBe(1); + } + }); + + test('Multiple boolean-like keys in same nested object', () => { + const data = { wrapper: { t: 1, f: 2, true: 3, false: 4, null: 5 } }; + const encoded = encode(data); + const decoded = decode(encoded) as any; + expect(decoded.wrapper.t).toBe(1); + expect(decoded.wrapper.f).toBe(2); + expect(decoded.wrapper.true).toBe(3); + expect(decoded.wrapper.false).toBe(4); + expect(decoded.wrapper.null).toBe(5); + }); + + test('Nested object with boolean-like key round-trips', () => { + const data = { a: { b: { c: { d: { e: { f: 1 } } } } } }; + const encoded = encode(data); + const decoded = decode(encoded) as any; + expect(decoded).toEqual(data); + const inner = decoded?.a?.b?.c?.d?.e; + expect(Object.keys(inner)).toContain('f'); + expect(Object.keys(inner)).not.toContain('false'); + }); + + test('Multiple sibling objects each with a boolean-like key round-trip', () => { + const data = { a: { f: 1 }, b: { t: 2 }, c: { null: 3 } }; + const encoded = encode(data); + const decoded = decode(encoded) as any; + expect(Object.keys(decoded.a)).toContain('f'); + expect(decoded.a.f).toBe(1); + expect(Object.keys(decoded.b)).toContain('t'); + expect(decoded.b.t).toBe(2); + expect(Object.keys(decoded.c)).toContain('null'); + expect(decoded.c.null).toBe(3); + }); +}); diff --git a/src/__tests__/dict-header-quoting.test.ts b/src/__tests__/dict-header-quoting.test.ts new file mode 100644 index 0000000..4db9b52 --- /dev/null +++ b/src/__tests__/dict-header-quoting.test.ts @@ -0,0 +1,59 @@ +/** + * Tests for dictionary header value quoting. + * + * Verifies the fix for a bug where dictionary compression header values + * containing commas or colons were truncated at the delimiter, causing + * incorrect round-trips. + */ + +import { encode, decode } from '../index'; + +describe('Dictionary header value quoting', () => { + test('Dictionary values with commas round-trip correctly', () => { + const data = [ + { Car: 'Toyota Celica ST 185', Year: '1996' }, + { Car: 'Toyota Corolla, AWD', Year: '1997' }, + { Car: 'Toyota Corolla, AWD', Year: '1998' }, + { Car: 'Toyota Corolla, AWD', Year: '1999' }, + { Car: 'Toyota Corolla, AWD', Year: '2000' }, + { Car: 'Toyota Corolla, AWD', Year: '2001' }, + ]; + const encoded = encode(data); + const decoded = decode(encoded) as any[]; + expect(decoded[0].Car).toBe('Toyota Celica ST 185'); + expect(decoded[1].Car).toBe('Toyota Corolla, AWD'); + expect(decoded[1].Year).toBe('1997'); + }); + + test('Dictionary values with colons round-trip correctly', () => { + const data = [ + { key: 'value:one', other: 'a' }, + { key: 'value:one', other: 'b' }, + { key: 'value:one', other: 'c' }, + { key: 'value:one', other: 'd' }, + { key: 'value:one', other: 'e' }, + { key: 'value:two', other: 'f' }, + ]; + const encoded = encode(data); + const decoded = decode(encoded) as any[]; + expect(decoded[0].key).toBe('value:one'); + expect(decoded[5].key).toBe('value:two'); + }); + + test('All rows preserved after dictionary decompression', () => { + const data = [ + { Car: 'Toyota Celica ST 185', Year: '1996' }, + { Car: 'Toyota Corolla, AWD', Year: '1997' }, + { Car: 'Toyota Corolla, AWD', Year: '1998' }, + { Car: 'Toyota Corolla, AWD', Year: '1999' }, + { Car: 'Toyota Corolla, AWD', Year: '2000' }, + { Car: 'Toyota Corolla, AWD', Year: '2001' }, + ]; + const encoded = encode(data); + const decoded = decode(encoded) as any[]; + expect(decoded).toHaveLength(data.length); + for (let i = 0; i < data.length; i++) { + expect(decoded[i]).toEqual(data[i]); + } + }); +}); diff --git a/src/core/decoder.ts b/src/core/decoder.ts index 40b5053..4de8857 100644 --- a/src/core/decoder.ts +++ b/src/core/decoder.ts @@ -1,6 +1,6 @@ import { TABLE_MARKER, META_SEPARATOR, MAX_DOCUMENT_SIZE, MAX_LINE_LENGTH, MAX_ARRAY_LENGTH, MAX_OBJECT_KEYS, MAX_NESTING_DEPTH } from './constants'; import { ZonDecodeError } from './exceptions'; -import { parseValue } from './utils'; +import { parseValue, parseKey } from './utils'; import { extractVersion, stripVersion, type ZonDocumentMetadata } from './versioning'; export interface DecodeOptions { @@ -94,7 +94,7 @@ export class ZonDecoder { const dictMatch = trimmedLine.match(/^([\w\.]+)\[(\d+)\]:(.+)$/); if (dictMatch && !trimmedLine.startsWith(TABLE_MARKER)) { const [, col, , vals] = dictMatch; - pendingDictionaries.set(col, vals.split(',')); + pendingDictionaries.set(col, this._splitByDelimiter(vals, ',').map(v => this._parseValue(v))); continue; } @@ -612,7 +612,7 @@ export class ZonDecoder { continue; } - const key = this._parseValue(keyStr); + const key = this._parseKey(keyStr); const val = this._parseZonNode(valStr, depth + 1); obj[key] = val; } @@ -745,7 +745,7 @@ export class ZonDecoder { // Handle implicit array item starting with dash in value? // No, _parseZonNode handles that recursively. - const key = this._parseValue(keyStr); + const key = this._parseKey(keyStr); const val = this._parseZonNode(valStr, depth + 1); obj[key] = val; } @@ -823,7 +823,7 @@ export class ZonDecoder { const colonIdx = this._findDelimiter(item, ':'); const keyStr = item.substring(0, colonIdx).trim(); const valStr = item.substring(colonIdx + 1).trim(); - const key = this._parseValue(keyStr); + const key = this._parseKey(keyStr); const val = this._parseZonNode(valStr, depth + 1); obj[key] = val; } @@ -856,7 +856,7 @@ export class ZonDecoder { const colonIdx = this._findDelimiter(line.trim(), ':'); const keyStr = line.substring(0, colonIdx).trim(); const valStr = line.substring(colonIdx + 1).trim(); - const key = this._parseValue(keyStr); + const key = this._parseKey(keyStr); const val = this._parseZonNode(valStr, depth + 1); obj[key] = val; } @@ -1043,6 +1043,10 @@ export class ZonDecoder { return parsed; } + private _parseKey(val: string): string { + return parseKey(val); + } + /** diff --git a/src/core/encoder.ts b/src/core/encoder.ts index 73e769c..0e59b07 100644 --- a/src/core/encoder.ts +++ b/src/core/encoder.ts @@ -449,7 +449,9 @@ export class ZonEncoder { const lines: string[] = []; for (const [col, values] of dictionaries) { - lines.push(`${col}[${values.length}]:${values.join(',')}`); + // Quote dictionary values that contain special characters + const formattedValues = values.map(v => this._formatValue(v)); + lines.push(`${col}[${values.length}]:${formattedValues.join(',')}`); } const dictCols = Array.from(dictionaries.keys()); @@ -628,7 +630,8 @@ export class ZonEncoder { const items: string[] = []; for (const k of keys) { let kStr = String(k); - if (/[,:\{\}\[\]"]/.test(kStr)) { + // Quote keys with special chars OR boolean/null keywords + if (/[,:\{\}\[\]"]/.test(kStr) || /^(true|false|t|f|null|none|nil)$/i.test(kStr)) { kStr = JSON.stringify(kStr); } diff --git a/src/core/utils.ts b/src/core/utils.ts index 512b133..d67f671 100644 --- a/src/core/utils.ts +++ b/src/core/utils.ts @@ -21,6 +21,41 @@ export function quoteString(s: string): string { return `"${zon}"`; } +/** + * Parses a ZON dictionary key string. + * + * Unlike parseValue, this does NOT convert boolean keywords (t, f, true, false) + * or null keywords to their typed equivalents. Keys are always strings. + * + * @param val - The key string to parse + * @returns The parsed key as a string + */ +export function parseKey(val: string): string { + const trimmed = val.trim(); + + if (trimmed.startsWith('"')) { + try { + return JSON.parse(trimmed); + } catch { + if (trimmed.endsWith('"')) { + const inner = trimmed.slice(1, -1); + const fixed = inner.replace(/""/g, '\\"'); + try { + return JSON.parse(`"${fixed}"`); + } catch { + return trimmed.slice(1, -1).replace(/""/g, '"'); + } + } + } + } + + if (trimmed.startsWith("'") && trimmed.endsWith("'")) { + return trimmed.slice(1, -1); + } + + return trimmed; +} + /** * Parses a ZON value string into its corresponding primitive type. * Handles booleans, nulls, numbers, arrays, objects, and quoted strings.