diff --git a/Lib/Tokenizer.wl b/Lib/Tokenizer.wl index b801686..206554e 100644 --- a/Lib/Tokenizer.wl +++ b/Lib/Tokenizer.wl @@ -1,40 +1,38 @@ (* ::Package:: *) +(* ::Input:: *) +(*StringMatchQ["&",RegularExpression["[^\\{\\}]*"]]*) + + BeginPackage["SMML`Tokenizer`"]; +RE=RegularExpression; rep[pat_]:=rep[pat,","~~" "...]; rep[pat_,sep_]:=pat~~(sep~~pat)...; -unsigned=DigitCharacter..; -integer=""|"-"|"+"~~unsigned; -chordCodeTok[str_]:=StringCases[str, - StringExpression[ - ntt:LetterCharacter~~"\t".., - cmt:Shortest[__]~~"\t".., - pts:rep[integer] - ]:><| - "Notation"->ntt, - "Comment"->cmt, - "Pitches"->StringSplit[pts,","~~" "...] - |> -][[1]]; +repRegex[pat_]:=repRegex[pat,", *"]; +repRegex[pat_,sep_]:=RE[pat<>"("<>sep<>pat<>")*"]; +sel[pat_]:=RE["("<>pat[[1]]<>")?"]; +or[pat__]:=RE["("<>StringRiffle[#[[1]]&/@{pat},"|"]<>")"]; +unsigned=RE["\\d+"]; +signed=RE["[\\+\\-]\\d+"]; +integer=RE["[\\+\\-]?\\d+"]; -intPsb=""|integer; -chordOpCode=rep["["~~intPsb|(intPsb~~";"~~intPsb)~~"]"~~intPsb]; -chordOpCodeTok[str_]:=StringCases[line, +chordCodeTok[str_]:=StringCases[str, StringExpression[ ntt:LetterCharacter~~"\t".., cmt:Shortest[__]~~"\t".., - pts:chordOpCode + pts:repRegex["(([\\+\\-]?\\d+)|(\\[([\\+\\-]?\\d+)?(;([\\+\\-]?\\d+)?)?\\]([\\-\\+]\\d+)?))"] ]:><| "Notation"->ntt, "Comment"->cmt, "Pitches"->StringCases[pts,{ - "["~~pos:intPsb~~"]"~~sft:intPsb:>{ + pit:integer:>{1,1,ToExpression[pit]}, + "["~~pos:sel[integer]~~"]"~~sft:sel[signed]:>{ If[pos=="",1,ToExpression[pos]], If[pos=="",-1,ToExpression[pos]], If[sft=="",0,ToExpression[sft]] }, - "["~~pos1:intPsb~~";"~~pos2:intPsb~~"]"~~sft:intPsb:>{ + "["~~pos1:sel[integer]~~";"~~pos2:sel[integer]~~"]"~~sft:sel[signed]:>{ If[pos1=="",1,ToExpression[pos1]], If[pos2=="",-1,ToExpression[pos2]], If[sft=="",0,ToExpression[sft]] @@ -67,7 +65,7 @@ postOperator="``"|"`"|""; volOperator=(">"|":")...; pitOperator=("#"|"b"|"'"|",")...; durOperator=("."|"-"|"_"|"=")...; -scaleDegree="0"|"1"|"2"|"3"|"4"|"5"|"6"|"7"|"%"|"x"; +scaleDegree=RE["[0-7%x]"]; number=integer~~""|("."~~unsigned); expression=(integer~~""|("/"|"."~~unsigned))|("Log2("~~unsigned~~")"); @@ -82,6 +80,7 @@ orderTok=Union@@StringCases[#,{ n:integer~~"~"~~m:integer:>Range[ToExpression@n,ToExpression@m], n:integer:>{ToExpression@n} }]&; +notationPadded=RE["[&\\|\\s\\^\\*]*"]; notationPatt=Alternatives[ "+"|"ToCoda"|"Coda"|"s"|"Segno"|"DC"|"DaCapo"|"DS"|"DaSegno", "||:"|":||"|("["~~orderListP~~".]"), @@ -107,12 +106,11 @@ notationTok=StringCases[{ typeHead="$"|"%"|"&"|"!"|"@"; typeName=<|"$"->"String","%"->"Expression","@"->"Subtrack","&"->"Object","!"->"Number"|>; -contextList={"End","ChordNotation","ChordOperator","Function"}; +contextList={"End","Chord","Function"}; settingList={"RecursionLimit","StaffDisplay","ForcedSpace"}; -syntaxTemplate=<|"ChordNotation"->{},"ChordOperator"->{},"Function"->{}|>; +syntaxTemplate=<|"Chord"->{},"Function"->{}|>; syntaxTags=<| - "ChordNotation"->"Notation", - "ChordOperator"->"Notation", + "Chord"->"Notation", "Function"->{"Name","Syntax"} |>; blankLineQ[str_]:=Or[ @@ -134,7 +132,7 @@ Tokenizer[filepath_]:=Block[ source,command,value, syntax=syntaxTemplate, - chordNotation,chordOperator, + chordPatt, pitch,pitches,note,pitchTok, objectPatt,objectTok, functionPatt,functionPattList, @@ -167,7 +165,7 @@ Tokenizer[filepath_]:=Block[ ][line]; If[contextData!={}, syntax[[context]]=Union[syntax[[context]],contextData[[All,syntaxTags[[context]]]]]; - AppendTo[library,<|"Type"->context,"Storage"->"Internal","Data"->contextData|>]; + AppendTo[library,<|"Type"->context,"Data"->contextData|>]; contextData={}; ]; Switch[command, @@ -185,10 +183,9 @@ Tokenizer[filepath_]:=Block[ ]]; Do[ syntax[[item]]=Union[syntax[[item]],source[["Syntax",item]]], - {item,{"ChordNotation","ChordOperator","Function"}}]; + {item,{"Chord","Function"}}]; AppendTo[library,<| "Type"->"Package", - "Storage"->"External", "Path"->StringTake[value,{2,-2}], "Content"->source[["Tokenizer","Library"]] |>], @@ -198,10 +195,8 @@ Tokenizer[filepath_]:=Block[ (* code *) Switch[context, - "ChordNotation", + "Chord", AppendTo[contextData,chordCodeTok[line]], - "ChordOperator", - AppendTo[contextData,chordOpCodeTok[line]], "Function", lineCount++; While[lineCount<=Length@rawData&&!StringMatchQ[line,functionCode], @@ -229,22 +224,19 @@ Tokenizer[filepath_]:=Block[ If[StringTake[filepath,-4]==".smp",Return[return]]; (* pitch *) - chordNotation=""|Alternatives@@syntax[["ChordNotation"]]; - chordOperator=Alternatives@@syntax[["ChordOperator"]]...; - pitch=scaleDegree~~pitOperator~~chordNotation~~chordOperator...; - pitches="["~~pitch..~~"]"~~pitOperator; - note=preOperator~~pitch|pitches~~volOperator~~durOperator~~postOperator; + chordPatt=RE["["<>syntax[["Chord"]]<>"]*"]; + pitch=scaleDegree~~pitOperator~~chordPatt; + pitches=pitch|("["~~pitch..~~"]"); + note=preOperator~~pitches~~pitOperator~~volOperator~~durOperator~~postOperator; pitchTok=StringCases[ StringExpression[ sd:scaleDegree, po:pitOperator, - cn:chordNotation, - co:chordOperator + ch:chordPatt ]:><| "ScaleDegree"->sd, "PitchOperators"->po, - "ChordNotations"->cn, - "ChordOperators"->co + "Chord"->ch |> ]; @@ -298,8 +290,8 @@ Tokenizer[filepath_]:=Block[ }][[1]]&}; (* function simplified *) - object=("{"~~subtrack~~"}")|(notationPatt...~~note~~notationPatt...); - typePatt=<|"$"->string,"%"->expression,"&"->object,"!"->number,"@"->subtrack|>; + object=("{"~~subtrack~~"}")|(notationPadded~~note~~notationPadded); + typePatt=<|"$"->string,"%"->expression,"&"->object,"!"->number,"@"->Shortest[subtrack]|>; typeTok[type_,str_]:=Switch[type, "String",<|"Type"->"String","Content"->str|>, "Expression",<|"Type"->"Expression","Content"->str|>, @@ -418,16 +410,8 @@ EndPackage[]; (* ::Input:: *) -(*Contexts["SMML`*"]*) - - -(* ::Input:: *) -(*Export[NotebookDirectory[]<>"test.json",%];*) - - -(* ::Input:: *) -(*Tokenizer[NotebookDirectory[]<>"test.sml"][["Tokenizer","Sections"]]*) +(*Tokenizer[NotebookDirectory[]<>"test.sml"][["Tokenizer","Sections",1,"Tracks",1,"Content"(*,All,"Type"*)]]*) (* ::Input:: *) -(*Export[NotebookDirectory[]<>"test4.json",Tokenizer[NotebookDirectory[]<>"test4.sml"][["Tokenizer"]]];//Timing*) +(*Export[NotebookDirectory[]<>"test5.json",Tokenizer[NotebookDirectory[]<>"test5.sml"][["Tokenizer"]]];//Timing*) diff --git a/Lib/parser.wl b/Lib/parser.wl index 029f35b..7a47315 100644 --- a/Lib/parser.wl +++ b/Lib/parser.wl @@ -375,7 +375,7 @@ parse[tokenizer_,sections_]:=Module[ (* ::Input:: *) (*AudioStop[];AudioPlay[#[[2]]]&@*) (*EchoFunction["time: ",#[[1]]&]@*) -(*Timing[integrate[#MusicClips,#Effects]&[parse[QYS`Tokenize[localPath<>"Songs\\Touhou\\Deep_Mountain.qys"],2]]];*) +(*Timing[integrate[#MusicClips,#Effects]&[parse[QYS`Tokenize[localPath<>"Songs\\Touhou\\Deep_Mountain.qys"],3]]];*) (* ::Input:: *) diff --git a/Lib/std/Standard.smp b/Lib/std/Standard.smp index 668ddf9..924f16f 100644 --- a/Lib/std/Standard.smp +++ b/Lib/std/Standard.smp @@ -4,15 +4,12 @@ # Include "./Global.smp" # Include "./Function.smp" -# ChordNotation +# Chord M Major 0, 4, 7 m Minor 0, 3, 7 a Augmented 0, 4, 8 d Diminished 0, 3, 6 - -# ChordOperator - t tres [], [1]+3 T tres+ [], [1]+4 q quattuor [], [1]+5 diff --git a/Songs/Touhou/Deep_Mountain.qys b/Songs/Touhou/Deep_Mountain.qys index 7df7e74..ac95938 100644 --- a/Songs/Touhou/Deep_Mountain.qys +++ b/Songs/Touhou/Deep_Mountain.qys @@ -87,7 +87,7 @@ 6_6=6_.66_5_6_|3-0_3_3_5_|6_6=6_.6(^7)1'_7_5_|3-0_3_3_5_\ 6_6=6_.66_5_6_|3-0_3_3_5_|6_6=6_.6(^7)1'_7_5_|6-0-| -<0.8>\ +\ 6_6=6_.66_5_6_|3-0_3_3_5_|6_6=6_.6(^7)1'_7_5_|3-0_3_3_5_\ 6_6=6_.66_5_6_|3-0_3_3_5_|6_6=6_.6(^7)1'_7_5_|6-0_1_1_2_\ 3_3=3_.33_2_3_|6,-0_1_1_2_|3_3=3_.3(^5)6_5_2_|1-0_1_1_2_\ diff --git a/Songs/Touhou/TH11-Chireiden/Hana_ni_Kaze.qys b/Songs/Touhou/TH11-Chireiden/Hana_ni_Kaze.qys index 2bf9d99..6cbd03f 100644 --- a/Songs/Touhou/TH11-Chireiden/Hana_ni_Kaze.qys +++ b/Songs/Touhou/TH11-Chireiden/Hana_ni_Kaze.qys @@ -135,7 +135,7 @@ 4.%.%-%%%|5.%.%-%%%|6.%.%-%%%|6.%.%-%%-\ 4.%.%-%%%|5.%.%-%%%|5#.%.%-%%%|6.%.%-%%%\ 4.%.%-%%%|5.%.%-%%%|6.%.%-%%%|6.%.%-%%-\ -{<0.4>[4o,6],13[14]----|5o,,7,25-257|5#o,,7,2[35#]----|6o,,136-316,\ +{<0.4>[4o6,],13[14]----|5o,,7,25-257|5#o,,7,2[35#]----|6o,,136-316,\ 4o,,6,1[36]----|5o,,7,25-27,5,_[6,1]_|[4,,46]-^|%-}| <0.5>\ @@ -355,7 +355,7 @@ 4.%.%-%%%|5.%.%-%%%|5#.%.%-%%%|6.%.%-%%%\ 4.%.%-%%%|5.%.%-%%%|6.%.%-%%%|6.%.%-%%-\ }\ -{<0.4>[4o,6],13[14]----|5o,,7,25-257|5#o,,7,2[35#]----|6o,,136-316,\ +{<0.4>[4,o6],13[14]----|5o,,7,25-257|5#o,,7,2[35#]----|6o,,136-316,\ 4o,,6,1[36]----|5o,,7,25-27,5,|[6,1]-^|%-}| <0.5>{2*\ diff --git a/Songs/Touhou/TH11-Chireiden/Taeta_Hashi.qys b/Songs/Touhou/TH11-Chireiden/Taeta_Hashi.qys index dbaa4f4..72d49ea 100644 --- a/Songs/Touhou/TH11-Chireiden/Taeta_Hashi.qys +++ b/Songs/Touhou/TH11-Chireiden/Taeta_Hashi.qys @@ -108,9 +108,9 @@ xxxx|xxxx|xxxx|xxxx|| 0---|0---|0---|0---\ 0---|0---|0---|0---\ 0---|0---|0---|0---\ -0---|0---|4#o`_[61'#]o`_5#o`_[61'#]o`_4#o`_[61'#]o`__4#o`__4#o`_[61'#]o`_|4#o`_[61'#]o`_5#o`_[61'#]o`_4#o`_[61'#]o`__4#o`__4#o`_[61'#]o`_\ -4#o`_[61'#]o`_5#o`_[61'#]o`_4#o`_[61'#]o`__4#o`__4#o`_[61'#]o`_|4#o`_[61'#]o`_5#o`_[61'#]o`_4#o`_[61'#]o`__4#o`__4#o`_[61'#]o`_\ -<1=C,>4#o`_[61'#]o`_5#o`_[61'#]o`_4#o`_[61'#]o`__4#o`__4#o`_[61'#]o`_|4#o`_[61'#]o`_5#o`_[61'#]o`_4#o`_[61'#]o`__4#o`__4#o`_[61'#]o`_\ +0---|0---|4#o`_[61'#]`_5#o`_[61'#]`_4#o`_[61'#]`__4#o`__4#o`_[61'#]`_|4#o`_[61'#]`_5#o`_[61'#]`_4#o`_[61'#]`__4#o`__4#o`_[61'#]`_\ +4#o`_[61'#]`_5#o`_[61'#]`_4#o`_[61'#]`__4#o`__4#o`_[61'#]`_|4#o`_[61'#]`_5#o`_[61'#]`_4#o`_[61'#]`__4#o`__4#o`_[61'#]`_\ +<1=C,>4#o`_[61'#]`_5#o`_[61'#]`_4#o`_[61'#]`__4#o`__4#o`_[61'#]`_|4#o`_[61'#]`_5#o`_[61'#]`_4#o`_[61'#]`__4#o`__4#o`_[61'#]`_\ <1=C>4#po`_6po`_5#po`_6po`_4#po`_6po`__4#po`__4#po`_6po`_|4#po`_6po`_5#po`_6po`_4#po-\ 4,#o---|[35#]---|[26]--[261'#]|1#M--[5#6]\ [37]-3o-|[24#]'-[27]2o|[1#3]'---|[74'#]-[15]'#-\