class b{options;tokensStable=[];tokensUnstable=[];lastUnstableCodeChunk="";lastStableGrammarState;constructor(t){this.options=t}async enqueue(t){const a=(this.lastUnstableCodeChunk+t).split(`
`),s=[];let e=[];const n=this.tokensUnstable.length;return a.forEach((o,i)=>{const r=i===a.length-1,l=this.options.highlighter.codeToTokens(o,{...this.options,grammarState:this.lastStableGrammarState}),h=l.tokens[0];r||h.push({content:`
`,offset:0}),r?(e=h,this.lastUnstableCodeChunk=o):(this.lastStableGrammarState=l.grammarState,s.push(...h))}),this.tokensStable.push(...s),this.tokensUnstable=e,{recall:n,stable:s,unstable:e}}close(){const t=this.tokensUnstable;return this.tokensUnstable=[],this.lastUnstableCodeChunk="",this.lastStableGrammarState=void 0,{stable:t}}clear(){this.tokensStable=[],this.tokensUnstable=[],this.lastUnstableCodeChunk="",this.lastStableGrammarState=void 0}clone(){const t=new b(this.options);return t.lastUnstableCodeChunk=this.lastUnstableCodeChunk,t.tokensUnstable=this.tokensUnstable,t.tokensStable=this.tokensStable,t.lastStableGrammarState=this.lastStableGrammarState,t}}class u extends TransformStream{tokenizer;options;constructor(t){const a=new b(t),{allowRecalls:s=!1}=t;super({async transform(e,n){const{stable:o,unstable:i,recall:r}=await a.enqueue(e);s&&r>0&&n.enqueue({recall:r});for(const l of o)n.enqueue(l);if(s)for(const l of i)n.enqueue(l)},async flush(e){const{stable:n}=a.close();if(!s)for(const o of n)e.enqueue(o)}}),this.tokenizer=a,this.options=t}}export{u as CodeToTokenTransformStream,b as ShikiStreamTokenizer};
