fix: fix list with tabs

src/Tokenizer.ts

@@ -3,6 +3,7 @@ import {
   rtrim,
   splitCells,
   findClosingBracket,
+  expandTabs,
 } from './helpers.ts';
 import type { Rules } from './rules.ts';
 import type { _Lexer } from './Lexer.ts';
@@ -267,7 +268,7 @@ export class _Tokenizer<ParserOutput = string, RendererOutput = string> {
         raw = cap[0];
         src = src.substring(raw.length);
 
-        let line = cap[2].split('\n', 1)[0].replace(this.rules.other.listReplaceTabs, (t: string) => ' '.repeat(3 * t.length));
+        let line = expandTabs(cap[2].split('\n', 1)[0], cap[1].length);
         let nextLine = src.split('\n', 1)[0];
         let blankLine = !line.trim();
 
@@ -278,7 +279,7 @@ export class _Tokenizer<ParserOutput = string, RendererOutput = string> {
         } else if (blankLine) {
           indent = cap[1].length + 1;
         } else {
-          indent = cap[2].search(this.rules.other.nonSpaceChar); // Find first non-space char
+          indent = line.search(this.rules.other.nonSpaceChar); // Find first non-space char
           indent = indent > 4 ? 1 : indent; // Treat indented code blocks (> 4 spaces) as having only 1 indent
           itemContents = line.slice(indent);
           indent += cap[1].length;

src/helpers.ts

@@ -147,3 +147,20 @@ export function findClosingBracket(str: string, b: string) {
 
   return -1;
 }
+
+export function expandTabs(line: string, indent = 0) {
+  let col = indent;
+  let expanded = '';
+  for (const char of line) {
+    if (char === '\t') {
+      const added = 4 - (col % 4);
+      expanded += ' '.repeat(added);
+      col += added;
+    } else {
+      expanded += char;
+      col++;
+    }
+  }
+
+  return expanded;
+}

src/rules.ts

@@ -44,7 +44,6 @@ export const other = {
   blockquoteStart: /^ {0,3}>/,
   blockquoteSetextReplace: /\n {0,3}((?:=+|-+) *)(?=\n|$)/g,
   blockquoteSetextReplace2: /^ {0,3}>[ \t]?/gm,
-  listReplaceTabs: /^\t+/,
   listReplaceNesting: /^ {1,4}(?=( {4})*[^ ])/g,
   listIsTask: /^\[[ xX]\] +\S/,
   listReplaceTask: /^\[[ xX]\] +/,
@@ -158,7 +157,7 @@ const paragraph = edit(_paragraph)
   .replace('|table', '')
   .replace('blockquote', ' {0,3}>')
   .replace('fences', ' {0,3}(?:`{3,}(?=[^`\\n]*\\n)|~{3,})[^\\n]*\\n')
-  .replace('list', ' {0,3}(?:[*+-]|1[.)]) ') // only lists starting from 1 can interrupt
+  .replace('list', ' {0,3}(?:[*+-]|1[.)])[ \\t]') // only lists starting from 1 can interrupt
   .replace('html', '</?(?:tag)(?: +|\\n|/?>)|<(?:script|pre|style|textarea|!--)')
   .replace('tag', _tag) // pars can be interrupted by type (6) html blocks
   .getRegex();
@@ -202,7 +201,7 @@ const gfmTable = edit(
   .replace('blockquote', ' {0,3}>')
   .replace('code', '(?: {4}| {0,3}\t)[^\\n]')
   .replace('fences', ' {0,3}(?:`{3,}(?=[^`\\n]*\\n)|~{3,})[^\\n]*\\n')
-  .replace('list', ' {0,3}(?:[*+-]|1[.)]) ') // only lists starting from 1 can interrupt
+  .replace('list', ' {0,3}(?:[*+-]|1[.)])[ \\t]') // only lists starting from 1 can interrupt
   .replace('html', '</?(?:tag)(?: +|\\n|/?>)|<(?:script|pre|style|textarea|!--)')
   .replace('tag', _tag) // tables can be interrupted by type (6) html blocks
   .getRegex();
@@ -218,7 +217,7 @@ const blockGfm: Record<BlockKeys, RegExp> = {
     .replace('table', gfmTable) // interrupt paragraphs with table
     .replace('blockquote', ' {0,3}>')
     .replace('fences', ' {0,3}(?:`{3,}(?=[^`\\n]*\\n)|~{3,})[^\\n]*\\n')
-    .replace('list', ' {0,3}(?:[*+-]|1[.)]) ') // only lists starting from 1 can interrupt
+    .replace('list', ' {0,3}(?:[*+-]|1[.)])[ \\t]') // only lists starting from 1 can interrupt
     .replace('html', '</?(?:tag)(?: +|\\n|/?>)|<(?:script|pre|style|textarea|!--)')
     .replace('tag', _tag) // pars can be interrupted by type (6) html blocks
     .getRegex(),

test/specs/new/list_item_tabs.html

@@ -1,17 +1,64 @@
 <ul>
-  <li>A</li>
-  <li>B</li>
-  <li>C</li>
-  <li>D</li>
-  <li>E</li>
-  <li>F</li>
+<li>A</li>
+<li>B</li>
+<li>C</li>
+<li>D</li>
+<li>E</li>
+<li>F</li>
 </ul>
-
 <ol>
-  <li>A</li>
-  <li>B</li>
-  <li>C</li>
-  <li>D</li>
-  <li>E</li>
-  <li>F</li>
+<li>A</li>
+<li>B</li>
+<li>C</li>
+<li>D</li>
+<li>E</li>
+<li>F</li>
 </ol>
+<p>I am using spaces after the number:</p>
+<ol>
+<li>Some Text</li>
+<li>Some Text</li>
+<li>Some Text</li>
+</ol>
+<p>I am using tabs after the number:</p>
+<ol>
+<li>SomeText</li>
+<li>SomeText</li>
+<li>SomeText</li>
+</ol>
+<p>I am using tabs after the number and have a space:</p>
+<ol>
+<li>SomeText</li>
+<li>SomeText</li>
+<li>SomeText</li>
+</ol>
+<p>I am using space after the number and have a tab:</p>
+<ol>
+<li>SomeText</li>
+<li>SomeText</li>
+<li>SomeText</li>
+</ol>
+<p>I am using spaces after the bullet:</p>
+<ul>
+<li>Some Text</li>
+<li>Some Text</li>
+<li>Some Text</li>
+</ul>
+<p>I am using tabs after the bullet:</p>
+<ul>
+<li>SomeText</li>
+<li>SomeText</li>
+<li>SomeText</li>
+</ul>
+<p>I am using tabs after the bullet and have a space:</p>
+<ul>
+<li>SomeText</li>
+<li>SomeText</li>
+<li>SomeText</li>
+</ul>
+<p>I am using space after the bullet and have a tab:</p>
+<ul>
+<li>SomeText</li>
+<li>SomeText</li>
+<li>SomeText</li>
+</ul>

test/specs/new/list_item_tabs.md

@@ -11,3 +11,47 @@
 4.	D
 5.	E
 6.	F
+
+I am using spaces after the number:
+1.  Some Text
+3.  Some Text
+4.  Some Text
+
+I am using tabs after the number:
+1.	SomeText
+2.	SomeText
+3.	SomeText
+
+I am using tabs after the number and have a space:
+
+1.	 SomeText
+2.	 SomeText
+3.	 SomeText
+
+I am using space after the number and have a tab:
+
+1. 	SomeText
+2. 	SomeText
+3. 	SomeText
+
+I am using spaces after the bullet:
+-  Some Text
+-  Some Text
+-  Some Text
+
+I am using tabs after the bullet:
+-	SomeText
+-	SomeText
+-	SomeText
+
+I am using tabs after the bullet and have a space:
+
+-	 SomeText
+-	 SomeText
+-	 SomeText
+
+I am using space after the bullet and have a tab:
+
+- 	SomeText
+- 	SomeText
+- 	SomeText