fix:ambiguity around extensions

docs/USING_ADVANCED.md

@@ -73,9 +73,9 @@ console.log(marked.parse(markdownString));
 |smartypants (**removed**)|`boolean` |`false`  |v0.2.9   |Removed in v8.0.0 use [`marked-smartypants`](https://www.npmjs.com/package/marked-smartypants) to use "smart" typographic punctuation for things like quotes and dashes.|
 |xhtml (**removed**)|`boolean` |`false`  |v0.3.2   |Removed in v8.0.0 use [`marked-xhtml`](https://www.npmjs.com/package/marked-xhtml) to emit self-closing HTML tags for void elements (<br/>, <img/>, etc.) with a "/" as required by XHTML.|
 
-<h2 id="extensions">Known Extensions</h2>
+<h2 id="extensions">Known MarkedExtensions (Plugins)</h2>
 
-Marked can be extended using [custom extensions](/using_pro#extensions). This is a list of extensions that can be used with `marked.use(extension)`.
+Marked can be extended using [MarkedExtensions (plugins)](/using_pro#extensions). These are complete plugin packages that can be used with `marked.use(extension)`. They are different from [custom tokenizer and renderer extensions](/using_pro#extensions) which are individual tokenizer/renderer pairs that go inside the `tokenizerAndRendererExtensions` array.
 
 <!-- Keep this list ordered alphabetically by name -->
 

docs/USING_PRO.md

@@ -29,13 +29,13 @@ marked.use(extension2);
 marked.use(extension3);
 ```
 
-All options will overwrite those previously set, except for the following options which will be merged with the existing framework and can be used to change or extend the functionality of Marked: `renderer`, `tokenizer`, `hooks`, `walkTokens`, and `extensions`.
+All options will overwrite those previously set, except for the following options which will be merged with the existing framework and can be used to change or extend the functionality of Marked: `renderer`, `tokenizer`, `hooks`, `walkTokens`, and `tokenizerAndRendererExtensions` (or `extensions` for backward compatibility).
 
 * The `renderer`, `tokenizer`, and `hooks` options are objects with functions that will be merged into the built-in `renderer` and `tokenizer` respectively.
 
 * The `walkTokens` option is a function that will be called to post-process every token before rendering.
 
-* The `extensions` option is an array of objects that can contain additional custom `renderer` and `tokenizer` steps that will execute before any of the default parsing logic occurs.
+* The `tokenizerAndRendererExtensions` option (or `extensions` for backward compatibility) is an array of objects that can contain additional custom `renderer` and `tokenizer` steps that will execute before any of the default parsing logic occurs.
 
 Importantly, ensure that the extensions are only added to `marked` once (ie in the global scope of a regular JavaScript or TypeScript module). If they are added in a function that is called repeatedly, or in the JS for an HTML component in a library such as Svelte, your extensions will be added repeatedly, eventually causing a recursion error. If you cannot prevent the code from being run repeatedly, you should create a [Marked instance](/using_advanced#instance) so that your extensions are stored independently from the global instance Marked provides.
 
@@ -52,7 +52,7 @@ Before building your custom extensions, it is important to understand the compon
 4) The `parser` traverses the token tree and feeds each token into the appropriate `renderer`, and concatenates their outputs into the final HTML result.
 5) Each `renderer` receives a token and manipulates its contents to generate a segment of HTML.
 
-Marked provides methods for directly overriding the `renderer` and `tokenizer` for any existing token type, as well as inserting additional custom `renderer` and `tokenizer` functions to handle entirely custom syntax. For example, using `marked.use({renderer})` would modify a renderer, whereas `marked.use({extensions: [{renderer}]})` would add a new renderer. See the [custom extensions example](#custom-extensions-example) for insight on how to execute this.
+Marked provides methods for directly overriding the `renderer` and `tokenizer` for any existing token type, as well as inserting additional custom `renderer` and `tokenizer` functions to handle entirely custom syntax. For example, using `marked.use({renderer})` would modify a renderer, whereas `marked.use({tokenizerAndRendererExtensions: [{renderer}]})` would add a new renderer. See the [custom extensions example](#custom-extensions-example) for insight on how to execute this.
 
 ***
 
@@ -104,7 +104,7 @@ console.log(marked.parse('# heading+'));
 
 ```js
 marked.use({
- extensions: [{
+ tokenizerAndRendererExtensions: [{
     name: 'heading',
     renderer(token) {
       return /* ... */
@@ -392,9 +392,9 @@ console.log(marked.parse(`_The formula is $a_ b=c_ d$._`));
 
 ***
 
-<h2 id="extensions">Custom Extensions : <code>extensions</code></h2>
+<h2 id="extensions">Custom Tokenizer and Renderer Extensions : <code>tokenizerAndRendererExtensions</code></h2>
 
-You may supply an `extensions` array to the `options` object. This array can contain any number of `extension` objects, using the following properties:
+You may supply a `tokenizerAndRendererExtensions` array to the `options` object. This array can contain any number of `extension` objects, using the following properties:
 
 <dl>
 <dt><code><strong>name</strong></code></dt>

src/Instance.ts

@@ -74,7 +74,7 @@ export class Marked<ParserOutput = string, RendererOutput = string> {
   }
 
   use(...args: MarkedExtension<ParserOutput, RendererOutput>[]) {
-    const extensions: MarkedOptions<ParserOutput, RendererOutput>['extensions'] = this.defaults.extensions || { renderers: {}, childTokens: {} };
+    const extensions: MarkedOptions<ParserOutput, RendererOutput>['tokenizerAndRendererExtensions'] = this.defaults.tokenizerAndRendererExtensions || this.defaults.extensions || { renderers: {}, childTokens: {} };
 
     args.forEach((pack) => {
       // copy options to new object
@@ -84,8 +84,9 @@ export class Marked<ParserOutput = string, RendererOutput = string> {
       opts.async = this.defaults.async || opts.async || false;
 
       // ==-- Parse "addon" extensions --== //
-      if (pack.extensions) {
-        pack.extensions.forEach((ext) => {
+      const extensionsArray = pack.tokenizerAndRendererExtensions || pack.extensions;
+      if (extensionsArray) {
+        extensionsArray.forEach((ext) => {
           if (!ext.name) {
             throw new Error('extension name required');
           }
@@ -134,6 +135,8 @@ export class Marked<ParserOutput = string, RendererOutput = string> {
             extensions.childTokens[ext.name] = ext.childTokens;
           }
         });
+        opts.tokenizerAndRendererExtensions = extensions;
+        // Also set the deprecated property for backward compatibility
         opts.extensions = extensions;
       }
 

src/Lexer.ts

@@ -383,15 +383,15 @@ export class _Lexer<ParserOutput = string, RendererOutput = string> {
         continue;
       }
 
-      // em & strong
-      if (token = this.tokenizer.emStrong(src, maskedSrc, prevChar)) {
+      // code
+      if (token = this.tokenizer.codespan(src)) {
         src = src.substring(token.raw.length);
         tokens.push(token);
         continue;
       }
 
-      // code
-      if (token = this.tokenizer.codespan(src)) {
+      // em & strong
+      if (token = this.tokenizer.emStrong(src, maskedSrc, prevChar)) {
         src = src.substring(token.raw.length);
         tokens.push(token);
         continue;

src/MarkedOptions.ts

@@ -63,6 +63,14 @@ export interface MarkedExtension<ParserOutput = string, RendererOutput = string>
   /**
    * Add tokenizers and renderers to marked
    */
+  tokenizerAndRendererExtensions?:
+    | TokenizerAndRendererExtension<ParserOutput, RendererOutput>[]
+    | null;
+
+  /**
+   * Add tokenizers and renderers to marked
+   * @deprecated Use tokenizerAndRendererExtensions instead
+   */
   extensions?:
     | TokenizerAndRendererExtension<ParserOutput, RendererOutput>[]
     | null;
@@ -114,7 +122,7 @@ export interface MarkedExtension<ParserOutput = string, RendererOutput = string>
   walkTokens?: ((token: Token) => void | Promise<void>) | null;
 }
 
-export interface MarkedOptions<ParserOutput = string, RendererOutput = string> extends Omit<MarkedExtension<ParserOutput, RendererOutput>, 'hooks' | 'renderer' | 'tokenizer' | 'extensions' | 'walkTokens'> {
+export interface MarkedOptions<ParserOutput = string, RendererOutput = string> extends Omit<MarkedExtension<ParserOutput, RendererOutput>, 'hooks' | 'renderer' | 'tokenizer' | 'tokenizerAndRendererExtensions' | 'extensions' | 'walkTokens'> {
   /**
    * Hooks are methods that hook into some part of marked.
    */
@@ -133,7 +141,24 @@ export interface MarkedOptions<ParserOutput = string, RendererOutput = string> e
   tokenizer?: _Tokenizer<ParserOutput, RendererOutput> | null;
 
   /**
-   * Custom extensions
+   * Custom tokenizer and renderer extensions
+   */
+  tokenizerAndRendererExtensions?: null | {
+    renderers: {
+      [name: string]: RendererExtensionFunction<ParserOutput, RendererOutput>;
+    };
+    childTokens: {
+      [name: string]: string[];
+    };
+    inline?: TokenizerExtensionFunction[];
+    block?: TokenizerExtensionFunction[];
+    startInline?: TokenizerStartFunction[];
+    startBlock?: TokenizerStartFunction[];
+  };
+
+  /**
+   * Custom tokenizer and renderer extensions
+   * @deprecated Use tokenizerAndRendererExtensions instead
    */
   extensions?: null | {
     renderers: {

src/defaults.ts

@@ -8,6 +8,7 @@ export function _getDefaults<ParserOutput = string, RendererOutput = string>():
     async: false,
     breaks: false,
     extensions: null,
+    tokenizerAndRendererExtensions: null,
     gfm: true,
     hooks: null,
     pedantic: false,

src/rules.ts

@@ -267,7 +267,7 @@ const _punctuationOrSpaceGfmStrongEm = /(?!~)[\s\p{P}\p{S}]/u;
 const _notPunctuationOrSpaceGfmStrongEm = /(?:[^\s\p{P}\p{S}]|~)/u;
 
 // sequences em should skip over [title](link), `code`, <html>
-const blockSkip = /\[[^\[\]]*?\]\((?:\\[\s\S]|[^\\\(\)]|\((?:\\[\s\S]|[^\\\(\)])*\))*\)|`[^`]*?`|<(?! )[^<>]*?>/g;
+const blockSkip = /\[[^\[\]]*?\]\((?:\\[\s\S]|[^\\\(\)]|\((?:\\[\s\S]|[^\\\(\)])*\))*\)|`+[^`]*?`+|<(?! )[^<>]*?>/g;
 
 const emStrongLDelimCore = /^(?:\*+(?:((?!\*)punct)|[^\s*]))|^_+(?:((?!_)punct)|([^\s_]))/;
 

test/types/marked.ts

@@ -240,6 +240,11 @@ marked.use({
   extensions: [tokenizerExtension, rendererExtension, tokenizerAndRendererExtension]
 });
 
+// Test new property name
+marked.use({
+  tokenizerAndRendererExtensions: [tokenizerExtension, rendererExtension, tokenizerAndRendererExtension]
+});
+
 const asyncExtension: MarkedExtension = {
   async: true,
   async walkTokens(token) {

test/unit/issue-3776-backtick-precedence.test.js

@@ -0,0 +1,208 @@
+import { Marked } from '../../src/marked.ts';
+import assert from 'node:assert';
+import { describe, it } from 'node:test';
+
+describe('Issue #3776: Backtick precedence', () => {
+  describe('Single backticks', () => {
+    it('should handle single backticks with emphasis correctly', () => {
+      const marked = new Marked();
+      const input = '**text `**` more**';
+      const expected = '<strong>text <code>**</code> more</strong>';
+
+      const result = marked.parseInline(input);
+      assert.strictEqual(result, expected);
+    });
+
+    it('should handle single backticks with underscore emphasis', () => {
+      const marked = new Marked();
+      const input = '__text `__` more__';
+      const expected = '<strong>text <code>__</code> more</strong>';
+
+      const result = marked.parseInline(input);
+      assert.strictEqual(result, expected);
+    });
+  });
+
+  describe('Double backticks', () => {
+    it('should handle double backticks with emphasis correctly', () => {
+      const marked = new Marked();
+      const input = '**text ``**`` more**';
+      const expected = '<strong>text <code>**</code> more</strong>';
+
+      const result = marked.parseInline(input);
+      assert.strictEqual(result, expected);
+    });
+
+    it('should handle double backticks with nested single backticks', () => {
+      const marked = new Marked();
+      const input = '**text ``code with ` backtick`` more**';
+      const expected = '<strong>text <code>code with ` backtick</code> more</strong>';
+
+      const result = marked.parseInline(input);
+      assert.strictEqual(result, expected);
+    });
+
+    it('should handle double backticks with underscore emphasis', () => {
+      const marked = new Marked();
+      const input = '__text ``__`` more__';
+      const expected = '<strong>text <code>__</code> more</strong>';
+
+      const result = marked.parseInline(input);
+      assert.strictEqual(result, expected);
+    });
+  });
+
+  describe('Triple backticks', () => {
+    it('should handle triple backticks with emphasis correctly', () => {
+      const marked = new Marked();
+      const input = '**text ```**``` more**';
+      const expected = '<strong>text <code>**</code> more</strong>';
+
+      const result = marked.parseInline(input);
+      assert.strictEqual(result, expected);
+    });
+
+    it('should handle triple backticks with nested backticks', () => {
+      const marked = new Marked();
+      const input = '**text ```code with `` double backticks``` more**';
+      const expected = '<strong>text <code>code with `` double backticks</code> more</strong>';
+
+      const result = marked.parseInline(input);
+      assert.strictEqual(result, expected);
+    });
+  });
+
+  describe('Quadruple backticks', () => {
+    it('should handle quadruple backticks with emphasis correctly', () => {
+      const marked = new Marked();
+      const input = '**text ````**```` more**';
+      const expected = '<strong>text <code>**</code> more</strong>';
+
+      const result = marked.parseInline(input);
+      assert.strictEqual(result, expected);
+    });
+
+    it('should handle quadruple backticks with nested triple backticks', () => {
+      const marked = new Marked();
+      const input = '**text ````code with ``` triple backticks```` more**';
+      const expected = '<strong>text <code>code with ``` triple backticks</code> more</strong>';
+
+      const result = marked.parseInline(input);
+      assert.strictEqual(result, expected);
+    });
+  });
+
+  describe('Mixed emphasis and code combinations', () => {
+    it('should handle multiple code spans in emphasis', () => {
+      const marked = new Marked();
+      const input = '**start `code1` middle ``code2`` end**';
+      const expected = '<strong>start <code>code1</code> middle <code>code2</code> end</strong>';
+
+      const result = marked.parseInline(input);
+      assert.strictEqual(result, expected);
+    });
+
+    it('should handle emphasis inside code spans (should not process)', () => {
+      const marked = new Marked();
+      const input = '`**not bold**`';
+      const expected = '<code>**not bold**</code>';
+
+      const result = marked.parseInline(input);
+      assert.strictEqual(result, expected);
+    });
+
+    it('should handle complex nested scenarios', () => {
+      const marked = new Marked();
+      const input = '*italic `code **not bold**` more italic*';
+      const expected = '<em>italic <code>code **not bold**</code> more italic</em>';
+
+      const result = marked.parseInline(input);
+      assert.strictEqual(result, expected);
+    });
+
+    it('should handle links with code spans', () => {
+      const marked = new Marked();
+      const input = '[link with `code`](url)';
+      const expected = '<a href="url">link with <code>code</code></a>';
+
+      const result = marked.parseInline(input);
+      assert.strictEqual(result, expected);
+    });
+  });
+
+  describe('Regression tests for normal emphasis and code', () => {
+    it('should still handle normal emphasis correctly', () => {
+      const marked = new Marked();
+      const input = '**bold text**';
+      const expected = '<strong>bold text</strong>';
+
+      const result = marked.parseInline(input);
+      assert.strictEqual(result, expected);
+    });
+
+    it('should still handle normal code correctly', () => {
+      const marked = new Marked();
+      const input = '`code`';
+      const expected = '<code>code</code>';
+
+      const result = marked.parseInline(input);
+      assert.strictEqual(result, expected);
+    });
+
+    it('should handle emphasis and code separately', () => {
+      const marked = new Marked();
+      const input = '**bold** and `code`';
+      const expected = '<strong>bold</strong> and <code>code</code>';
+
+      const result = marked.parseInline(input);
+      assert.strictEqual(result, expected);
+    });
+
+    it('should handle nested emphasis correctly', () => {
+      const marked = new Marked();
+      const input = '**bold _italic_ bold**';
+      const expected = '<strong>bold <em>italic</em> bold</strong>';
+
+      const result = marked.parseInline(input);
+      assert.strictEqual(result, expected);
+    });
+  });
+
+  describe('Edge cases', () => {
+    it('should handle unmatched backticks', () => {
+      const marked = new Marked();
+      const input = '**text ` unmatched**';
+      const expected = '<strong>text ` unmatched</strong>';
+
+      const result = marked.parseInline(input);
+      assert.strictEqual(result, expected);
+    });
+
+    it('should handle empty code spans', () => {
+      const marked = new Marked();
+      const input = '**text `` `` more**';
+      const expected = '<strong>text <code> </code> more</strong>';
+
+      const result = marked.parseInline(input);
+      assert.strictEqual(result, expected);
+    });
+
+    it('should handle code spans at boundaries', () => {
+      const marked = new Marked();
+      const input = '``code`` **bold**';
+      const expected = '<code>code</code> <strong>bold</strong>';
+
+      const result = marked.parseInline(input);
+      assert.strictEqual(result, expected);
+    });
+
+    it('should handle code spans with HTML entities', () => {
+      const marked = new Marked();
+      const input = '**text `&lt;tag&gt;` more**';
+      const expected = '<strong>text <code>&amp;lt;tag&amp;gt;</code> more</strong>';
+
+      const result = marked.parseInline(input);
+      assert.strictEqual(result, expected);
+    });
+  });
+});