diff --git a/docs/USING_ADVANCED.md b/docs/USING_ADVANCED.md index ba758e56f5..c38e230cc1 100644 --- a/docs/USING_ADVANCED.md +++ b/docs/USING_ADVANCED.md @@ -73,9 +73,9 @@ console.log(marked.parse(markdownString)); |smartypants (**removed**)|`boolean` |`false` |v0.2.9 |Removed in v8.0.0 use [`marked-smartypants`](https://www.npmjs.com/package/marked-smartypants) to use "smart" typographic punctuation for things like quotes and dashes.| |xhtml (**removed**)|`boolean` |`false` |v0.3.2 |Removed in v8.0.0 use [`marked-xhtml`](https://www.npmjs.com/package/marked-xhtml) to emit self-closing HTML tags for void elements (<br/>, <img/>, etc.) with a "/" as required by XHTML.| -

Known Extensions

+

Known MarkedExtensions (Plugins)

-Marked can be extended using [custom extensions](/using_pro#extensions). This is a list of extensions that can be used with `marked.use(extension)`. +Marked can be extended using [MarkedExtensions (plugins)](/using_pro#extensions). These are complete plugin packages that can be used with `marked.use(extension)`. They are different from [custom tokenizer and renderer extensions](/using_pro#extensions) which are individual tokenizer/renderer pairs that go inside the `tokenizerAndRendererExtensions` array. diff --git a/docs/USING_PRO.md b/docs/USING_PRO.md index 57fcc15e0f..090cf48d81 100644 --- a/docs/USING_PRO.md +++ b/docs/USING_PRO.md @@ -29,13 +29,13 @@ marked.use(extension2); marked.use(extension3); ``` -All options will overwrite those previously set, except for the following options which will be merged with the existing framework and can be used to change or extend the functionality of Marked: `renderer`, `tokenizer`, `hooks`, `walkTokens`, and `extensions`. +All options will overwrite those previously set, except for the following options which will be merged with the existing framework and can be used to change or extend the functionality of Marked: `renderer`, `tokenizer`, `hooks`, `walkTokens`, and `tokenizerAndRendererExtensions` (or `extensions` for backward compatibility). * The `renderer`, `tokenizer`, and `hooks` options are objects with functions that will be merged into the built-in `renderer` and `tokenizer` respectively. * The `walkTokens` option is a function that will be called to post-process every token before rendering. -* The `extensions` option is an array of objects that can contain additional custom `renderer` and `tokenizer` steps that will execute before any of the default parsing logic occurs. +* The `tokenizerAndRendererExtensions` option (or `extensions` for backward compatibility) is an array of objects that can contain additional custom `renderer` and `tokenizer` steps that will execute before any of the default parsing logic occurs. Importantly, ensure that the extensions are only added to `marked` once (ie in the global scope of a regular JavaScript or TypeScript module). If they are added in a function that is called repeatedly, or in the JS for an HTML component in a library such as Svelte, your extensions will be added repeatedly, eventually causing a recursion error. If you cannot prevent the code from being run repeatedly, you should create a [Marked instance](/using_advanced#instance) so that your extensions are stored independently from the global instance Marked provides. @@ -52,7 +52,7 @@ Before building your custom extensions, it is important to understand the compon 4) The `parser` traverses the token tree and feeds each token into the appropriate `renderer`, and concatenates their outputs into the final HTML result. 5) Each `renderer` receives a token and manipulates its contents to generate a segment of HTML. -Marked provides methods for directly overriding the `renderer` and `tokenizer` for any existing token type, as well as inserting additional custom `renderer` and `tokenizer` functions to handle entirely custom syntax. For example, using `marked.use({renderer})` would modify a renderer, whereas `marked.use({extensions: [{renderer}]})` would add a new renderer. See the [custom extensions example](#custom-extensions-example) for insight on how to execute this. +Marked provides methods for directly overriding the `renderer` and `tokenizer` for any existing token type, as well as inserting additional custom `renderer` and `tokenizer` functions to handle entirely custom syntax. For example, using `marked.use({renderer})` would modify a renderer, whereas `marked.use({tokenizerAndRendererExtensions: [{renderer}]})` would add a new renderer. See the [custom extensions example](#custom-extensions-example) for insight on how to execute this. *** @@ -104,7 +104,7 @@ console.log(marked.parse('# heading+')); ```js marked.use({ - extensions: [{ + tokenizerAndRendererExtensions: [{ name: 'heading', renderer(token) { return /* ... */ @@ -392,9 +392,9 @@ console.log(marked.parse(`_The formula is $a_ b=c_ d$._`)); *** -

Custom Extensions : extensions

+

Custom Tokenizer and Renderer Extensions : tokenizerAndRendererExtensions

-You may supply an `extensions` array to the `options` object. This array can contain any number of `extension` objects, using the following properties: +You may supply a `tokenizerAndRendererExtensions` array to the `options` object. This array can contain any number of `extension` objects, using the following properties:
name
diff --git a/src/Instance.ts b/src/Instance.ts index df2d20f09c..2867a6a6db 100644 --- a/src/Instance.ts +++ b/src/Instance.ts @@ -74,7 +74,7 @@ export class Marked { } use(...args: MarkedExtension[]) { - const extensions: MarkedOptions['extensions'] = this.defaults.extensions || { renderers: {}, childTokens: {} }; + const extensions: MarkedOptions['tokenizerAndRendererExtensions'] = this.defaults.tokenizerAndRendererExtensions || this.defaults.extensions || { renderers: {}, childTokens: {} }; args.forEach((pack) => { // copy options to new object @@ -84,8 +84,9 @@ export class Marked { opts.async = this.defaults.async || opts.async || false; // ==-- Parse "addon" extensions --== // - if (pack.extensions) { - pack.extensions.forEach((ext) => { + const extensionsArray = pack.tokenizerAndRendererExtensions || pack.extensions; + if (extensionsArray) { + extensionsArray.forEach((ext) => { if (!ext.name) { throw new Error('extension name required'); } @@ -134,6 +135,8 @@ export class Marked { extensions.childTokens[ext.name] = ext.childTokens; } }); + opts.tokenizerAndRendererExtensions = extensions; + // Also set the deprecated property for backward compatibility opts.extensions = extensions; } diff --git a/src/Lexer.ts b/src/Lexer.ts index e38094578e..59e52b65e5 100644 --- a/src/Lexer.ts +++ b/src/Lexer.ts @@ -383,15 +383,15 @@ export class _Lexer { continue; } - // em & strong - if (token = this.tokenizer.emStrong(src, maskedSrc, prevChar)) { + // code + if (token = this.tokenizer.codespan(src)) { src = src.substring(token.raw.length); tokens.push(token); continue; } - // code - if (token = this.tokenizer.codespan(src)) { + // em & strong + if (token = this.tokenizer.emStrong(src, maskedSrc, prevChar)) { src = src.substring(token.raw.length); tokens.push(token); continue; diff --git a/src/MarkedOptions.ts b/src/MarkedOptions.ts index 6ef3353330..e0b9d62726 100644 --- a/src/MarkedOptions.ts +++ b/src/MarkedOptions.ts @@ -63,6 +63,14 @@ export interface MarkedExtension /** * Add tokenizers and renderers to marked */ + tokenizerAndRendererExtensions?: + | TokenizerAndRendererExtension[] + | null; + + /** + * Add tokenizers and renderers to marked + * @deprecated Use tokenizerAndRendererExtensions instead + */ extensions?: | TokenizerAndRendererExtension[] | null; @@ -114,7 +122,7 @@ export interface MarkedExtension walkTokens?: ((token: Token) => void | Promise) | null; } -export interface MarkedOptions extends Omit, 'hooks' | 'renderer' | 'tokenizer' | 'extensions' | 'walkTokens'> { +export interface MarkedOptions extends Omit, 'hooks' | 'renderer' | 'tokenizer' | 'tokenizerAndRendererExtensions' | 'extensions' | 'walkTokens'> { /** * Hooks are methods that hook into some part of marked. */ @@ -133,7 +141,24 @@ export interface MarkedOptions e tokenizer?: _Tokenizer | null; /** - * Custom extensions + * Custom tokenizer and renderer extensions + */ + tokenizerAndRendererExtensions?: null | { + renderers: { + [name: string]: RendererExtensionFunction; + }; + childTokens: { + [name: string]: string[]; + }; + inline?: TokenizerExtensionFunction[]; + block?: TokenizerExtensionFunction[]; + startInline?: TokenizerStartFunction[]; + startBlock?: TokenizerStartFunction[]; + }; + + /** + * Custom tokenizer and renderer extensions + * @deprecated Use tokenizerAndRendererExtensions instead */ extensions?: null | { renderers: { diff --git a/src/defaults.ts b/src/defaults.ts index 28314f367d..84762e53e5 100644 --- a/src/defaults.ts +++ b/src/defaults.ts @@ -8,6 +8,7 @@ export function _getDefaults(): async: false, breaks: false, extensions: null, + tokenizerAndRendererExtensions: null, gfm: true, hooks: null, pedantic: false, diff --git a/src/rules.ts b/src/rules.ts index 67dcb09ece..ee0df16fc3 100644 --- a/src/rules.ts +++ b/src/rules.ts @@ -267,7 +267,7 @@ const _punctuationOrSpaceGfmStrongEm = /(?!~)[\s\p{P}\p{S}]/u; const _notPunctuationOrSpaceGfmStrongEm = /(?:[^\s\p{P}\p{S}]|~)/u; // sequences em should skip over [title](link), `code`, -const blockSkip = /\[[^\[\]]*?\]\((?:\\[\s\S]|[^\\\(\)]|\((?:\\[\s\S]|[^\\\(\)])*\))*\)|`[^`]*?`|<(?! )[^<>]*?>/g; +const blockSkip = /\[[^\[\]]*?\]\((?:\\[\s\S]|[^\\\(\)]|\((?:\\[\s\S]|[^\\\(\)])*\))*\)|`+[^`]*?`+|<(?! )[^<>]*?>/g; const emStrongLDelimCore = /^(?:\*+(?:((?!\*)punct)|[^\s*]))|^_+(?:((?!_)punct)|([^\s_]))/; diff --git a/test/types/marked.ts b/test/types/marked.ts index 43712b6059..70c7d4b2ee 100644 --- a/test/types/marked.ts +++ b/test/types/marked.ts @@ -240,6 +240,11 @@ marked.use({ extensions: [tokenizerExtension, rendererExtension, tokenizerAndRendererExtension] }); +// Test new property name +marked.use({ + tokenizerAndRendererExtensions: [tokenizerExtension, rendererExtension, tokenizerAndRendererExtension] +}); + const asyncExtension: MarkedExtension = { async: true, async walkTokens(token) { diff --git a/test/unit/issue-3776-backtick-precedence.test.js b/test/unit/issue-3776-backtick-precedence.test.js new file mode 100644 index 0000000000..f0b3218a3f --- /dev/null +++ b/test/unit/issue-3776-backtick-precedence.test.js @@ -0,0 +1,208 @@ +import { Marked } from '../../src/marked.ts'; +import assert from 'node:assert'; +import { describe, it } from 'node:test'; + +describe('Issue #3776: Backtick precedence', () => { + describe('Single backticks', () => { + it('should handle single backticks with emphasis correctly', () => { + const marked = new Marked(); + const input = '**text `**` more**'; + const expected = 'text ** more'; + + const result = marked.parseInline(input); + assert.strictEqual(result, expected); + }); + + it('should handle single backticks with underscore emphasis', () => { + const marked = new Marked(); + const input = '__text `__` more__'; + const expected = 'text __ more'; + + const result = marked.parseInline(input); + assert.strictEqual(result, expected); + }); + }); + + describe('Double backticks', () => { + it('should handle double backticks with emphasis correctly', () => { + const marked = new Marked(); + const input = '**text ``**`` more**'; + const expected = 'text ** more'; + + const result = marked.parseInline(input); + assert.strictEqual(result, expected); + }); + + it('should handle double backticks with nested single backticks', () => { + const marked = new Marked(); + const input = '**text ``code with ` backtick`` more**'; + const expected = 'text code with ` backtick more'; + + const result = marked.parseInline(input); + assert.strictEqual(result, expected); + }); + + it('should handle double backticks with underscore emphasis', () => { + const marked = new Marked(); + const input = '__text ``__`` more__'; + const expected = 'text __ more'; + + const result = marked.parseInline(input); + assert.strictEqual(result, expected); + }); + }); + + describe('Triple backticks', () => { + it('should handle triple backticks with emphasis correctly', () => { + const marked = new Marked(); + const input = '**text ```**``` more**'; + const expected = 'text ** more'; + + const result = marked.parseInline(input); + assert.strictEqual(result, expected); + }); + + it('should handle triple backticks with nested backticks', () => { + const marked = new Marked(); + const input = '**text ```code with `` double backticks``` more**'; + const expected = 'text code with `` double backticks more'; + + const result = marked.parseInline(input); + assert.strictEqual(result, expected); + }); + }); + + describe('Quadruple backticks', () => { + it('should handle quadruple backticks with emphasis correctly', () => { + const marked = new Marked(); + const input = '**text ````**```` more**'; + const expected = 'text ** more'; + + const result = marked.parseInline(input); + assert.strictEqual(result, expected); + }); + + it('should handle quadruple backticks with nested triple backticks', () => { + const marked = new Marked(); + const input = '**text ````code with ``` triple backticks```` more**'; + const expected = 'text code with ``` triple backticks more'; + + const result = marked.parseInline(input); + assert.strictEqual(result, expected); + }); + }); + + describe('Mixed emphasis and code combinations', () => { + it('should handle multiple code spans in emphasis', () => { + const marked = new Marked(); + const input = '**start `code1` middle ``code2`` end**'; + const expected = 'start code1 middle code2 end'; + + const result = marked.parseInline(input); + assert.strictEqual(result, expected); + }); + + it('should handle emphasis inside code spans (should not process)', () => { + const marked = new Marked(); + const input = '`**not bold**`'; + const expected = '**not bold**'; + + const result = marked.parseInline(input); + assert.strictEqual(result, expected); + }); + + it('should handle complex nested scenarios', () => { + const marked = new Marked(); + const input = '*italic `code **not bold**` more italic*'; + const expected = 'italic code **not bold** more italic'; + + const result = marked.parseInline(input); + assert.strictEqual(result, expected); + }); + + it('should handle links with code spans', () => { + const marked = new Marked(); + const input = '[link with `code`](url)'; + const expected = 'link with code'; + + const result = marked.parseInline(input); + assert.strictEqual(result, expected); + }); + }); + + describe('Regression tests for normal emphasis and code', () => { + it('should still handle normal emphasis correctly', () => { + const marked = new Marked(); + const input = '**bold text**'; + const expected = 'bold text'; + + const result = marked.parseInline(input); + assert.strictEqual(result, expected); + }); + + it('should still handle normal code correctly', () => { + const marked = new Marked(); + const input = '`code`'; + const expected = 'code'; + + const result = marked.parseInline(input); + assert.strictEqual(result, expected); + }); + + it('should handle emphasis and code separately', () => { + const marked = new Marked(); + const input = '**bold** and `code`'; + const expected = 'bold and code'; + + const result = marked.parseInline(input); + assert.strictEqual(result, expected); + }); + + it('should handle nested emphasis correctly', () => { + const marked = new Marked(); + const input = '**bold _italic_ bold**'; + const expected = 'bold italic bold'; + + const result = marked.parseInline(input); + assert.strictEqual(result, expected); + }); + }); + + describe('Edge cases', () => { + it('should handle unmatched backticks', () => { + const marked = new Marked(); + const input = '**text ` unmatched**'; + const expected = 'text ` unmatched'; + + const result = marked.parseInline(input); + assert.strictEqual(result, expected); + }); + + it('should handle empty code spans', () => { + const marked = new Marked(); + const input = '**text `` `` more**'; + const expected = 'text more'; + + const result = marked.parseInline(input); + assert.strictEqual(result, expected); + }); + + it('should handle code spans at boundaries', () => { + const marked = new Marked(); + const input = '``code`` **bold**'; + const expected = 'code bold'; + + const result = marked.parseInline(input); + assert.strictEqual(result, expected); + }); + + it('should handle code spans with HTML entities', () => { + const marked = new Marked(); + const input = '**text `<tag>` more**'; + const expected = 'text &lt;tag&gt; more'; + + const result = marked.parseInline(input); + assert.strictEqual(result, expected); + }); + }); +}); \ No newline at end of file diff --git a/test_new_property.js b/test_new_property.js new file mode 100644 index 0000000000..a5780ea227 --- /dev/null +++ b/test_new_property.js @@ -0,0 +1,97 @@ +import { Marked } from './src/marked.ts'; + +// Test the backtick precedence fix +console.log('Testing backtick precedence fix:'); + +const testCases = [ + { + input: '**text `**` more**', + expected: 'text ** more', + description: 'Single backticks with emphasis - code takes precedence' + }, + { + input: '**text ``**`` more**', + expected: 'text ** more', + description: 'Double backticks with emphasis - code takes precedence' + }, + { + input: '**text ```**``` more**', + expected: 'text ** more', + description: 'Triple backticks with emphasis - code takes precedence' + }, + { + input: '**text ````**```` more**', + expected: 'text ** more', + description: 'Quadruple backticks with emphasis - code takes precedence' + }, + { + input: '__text ``__`` more__', + expected: 'text __ more', + description: 'Double backticks with underscore emphasis' + }, + { + input: '`**not bold**`', + expected: '**not bold**', + description: 'Code spans should prevent emphasis processing' + }, + { + input: '**start `code` end**', + expected: 'start code end', + description: 'Code within emphasis should work' + } +]; + +testCases.forEach(({ input, expected, description }) => { + try { + // Create a fresh instance for each test + const marked = new Marked(); + const result = marked.parseInline(input); + const passed = result === expected; + console.log(`${passed ? '✓' : '✗'} ${description}`); + if (!passed) { + console.log(` Input: ${input}`); + console.log(` Expected: ${expected}`); + console.log(` Got: ${result}`); + } + } catch (e) { + console.log(`✗ ${description} - Error: ${e.message}`); + } +}); + +// Test that normal cases still work +console.log('\nTesting regression cases:'); + +const regressionCases = [ + { + input: '**bold text**', + expected: 'bold text', + description: 'Normal emphasis should still work' + }, + { + input: '`code`', + expected: 'code', + description: 'Normal code should still work' + }, + { + input: '**bold** and `code`', + expected: 'bold and code', + description: 'Separate emphasis and code should work' + } +]; + +regressionCases.forEach(({ input, expected, description }) => { + try { + // Create a fresh instance for each test + const marked = new Marked(); + const result = marked.parseInline(input); + const passed = result === expected; + console.log(`${passed ? '✓' : '✗'} ${description}`); + if (!passed) { + console.log(` Input: ${input}`); + console.log(` Expected: ${expected}`); + console.log(` Got: ${result}`); + } + } catch (e) { + console.log(`✗ ${description} - Error: ${e.message}`); + } +}); \ No newline at end of file