From ef27bee00a8b0e0216af30956e3fe1b11159bfae Mon Sep 17 00:00:00 2001 From: Shigma <1700011071@pku.edu.cn> Date: Thu, 15 Feb 2018 15:07:17 +0800 Subject: [PATCH 1/3] Integrate type: ChordNotation and type: ChordOperator --- Lib/Tokenizer.wl | 74 +++++++++----------- Lib/parser.wl | 2 +- Lib/std/Standard.smp | 5 +- Songs/Touhou/Deep_Mountain.qys | 2 +- Songs/Touhou/TH11-Chireiden/Hana_ni_Kaze.qys | 4 +- 5 files changed, 39 insertions(+), 48 deletions(-) diff --git a/Lib/Tokenizer.wl b/Lib/Tokenizer.wl index b801686..7e6cb9a 100644 --- a/Lib/Tokenizer.wl +++ b/Lib/Tokenizer.wl @@ -1,40 +1,39 @@ (* ::Package:: *) +(* ::Input:: *) +(*StringMatchQ[",",RegularExpression["\\,"]]*) + + BeginPackage["SMML`Tokenizer`"]; +RE=RegularExpression; rep[pat_]:=rep[pat,","~~" "...]; rep[pat_,sep_]:=pat~~(sep~~pat)...; -unsigned=DigitCharacter..; -integer=""|"-"|"+"~~unsigned; -chordCodeTok[str_]:=StringCases[str, - StringExpression[ - ntt:LetterCharacter~~"\t".., - cmt:Shortest[__]~~"\t".., - pts:rep[integer] - ]:><| - "Notation"->ntt, - "Comment"->cmt, - "Pitches"->StringSplit[pts,","~~" "...] - |> -][[1]]; +repRegex[pat_]:=repRegex[pat,", *"]; +repRegex[pat_,sep_]:=RE[pat<>"("<>sep<>pat<>")*"]; +sel[pat_RegularExpression]:=RE["("<>pat[[1]]<>")?"]; +unsigned=RE["\\d+"]; +signed=RE["[\\+\\-]\\d+"]; +integer=RE["[\\+\\-]?\\d+"]; -intPsb=""|integer; -chordOpCode=rep["["~~intPsb|(intPsb~~";"~~intPsb)~~"]"~~intPsb]; -chordOpCodeTok[str_]:=StringCases[line, +chordCodePos=RE["([\\+\\-]?\\d+)?"]; +chordCodeSft=RE["([\\+\\-]\\d+)?"]; +chordCodeTok[str_]:=StringCases[str, StringExpression[ ntt:LetterCharacter~~"\t".., cmt:Shortest[__]~~"\t".., - pts:chordOpCode + pts:repRegex["(([\\+\\-]?\\d+)|(\\[([\\+\\-]?\\d+)?(;([\\+\\-]?\\d+)?)?\\]([\\-\\+]\\d+)?))"] ]:><| "Notation"->ntt, "Comment"->cmt, "Pitches"->StringCases[pts,{ - "["~~pos:intPsb~~"]"~~sft:intPsb:>{ + pit:integer:>{1,1,ToExpression[pit]}, + "["~~pos:sel[integer]~~"]"~~sft:sel[signed]:>{ If[pos=="",1,ToExpression[pos]], If[pos=="",-1,ToExpression[pos]], If[sft=="",0,ToExpression[sft]] }, - "["~~pos1:intPsb~~";"~~pos2:intPsb~~"]"~~sft:intPsb:>{ + "["~~pos1:sel[integer]~~";"~~pos2:sel[integer]~~"]"~~sft:sel[signed]:>{ If[pos1=="",1,ToExpression[pos1]], If[pos2=="",-1,ToExpression[pos2]], If[sft=="",0,ToExpression[sft]] @@ -82,6 +81,7 @@ orderTok=Union@@StringCases[#,{ n:integer~~"~"~~m:integer:>Range[ToExpression@n,ToExpression@m], n:integer:>{ToExpression@n} }]&; +notationPadded=Whitespace|"|"|"/"|"^"|"&"|"*"; notationPatt=Alternatives[ "+"|"ToCoda"|"Coda"|"s"|"Segno"|"DC"|"DaCapo"|"DS"|"DaSegno", "||:"|":||"|("["~~orderListP~~".]"), @@ -107,12 +107,11 @@ notationTok=StringCases[{ typeHead="$"|"%"|"&"|"!"|"@"; typeName=<|"$"->"String","%"->"Expression","@"->"Subtrack","&"->"Object","!"->"Number"|>; -contextList={"End","ChordNotation","ChordOperator","Function"}; +contextList={"End","Chord","Function"}; settingList={"RecursionLimit","StaffDisplay","ForcedSpace"}; -syntaxTemplate=<|"ChordNotation"->{},"ChordOperator"->{},"Function"->{}|>; +syntaxTemplate=<|"Chord"->{},"Function"->{}|>; syntaxTags=<| - "ChordNotation"->"Notation", - "ChordOperator"->"Notation", + "Chord"->"Notation", "Function"->{"Name","Syntax"} |>; blankLineQ[str_]:=Or[ @@ -134,7 +133,7 @@ Tokenizer[filepath_]:=Block[ source,command,value, syntax=syntaxTemplate, - chordNotation,chordOperator, + chordPatt, pitch,pitches,note,pitchTok, objectPatt,objectTok, functionPatt,functionPattList, @@ -185,7 +184,7 @@ Tokenizer[filepath_]:=Block[ ]]; Do[ syntax[[item]]=Union[syntax[[item]],source[["Syntax",item]]], - {item,{"ChordNotation","ChordOperator","Function"}}]; + {item,{"Chord","Function"}}]; AppendTo[library,<| "Type"->"Package", "Storage"->"External", @@ -198,10 +197,8 @@ Tokenizer[filepath_]:=Block[ (* code *) Switch[context, - "ChordNotation", + "Chord", AppendTo[contextData,chordCodeTok[line]], - "ChordOperator", - AppendTo[contextData,chordOpCodeTok[line]], "Function", lineCount++; While[lineCount<=Length@rawData&&!StringMatchQ[line,functionCode], @@ -229,22 +226,19 @@ Tokenizer[filepath_]:=Block[ If[StringTake[filepath,-4]==".smp",Return[return]]; (* pitch *) - chordNotation=""|Alternatives@@syntax[["ChordNotation"]]; - chordOperator=Alternatives@@syntax[["ChordOperator"]]...; - pitch=scaleDegree~~pitOperator~~chordNotation~~chordOperator...; - pitches="["~~pitch..~~"]"~~pitOperator; - note=preOperator~~pitch|pitches~~volOperator~~durOperator~~postOperator; + chordPatt=RE["["<>syntax[["Chord"]]<>"]*"]; + pitch=scaleDegree~~pitOperator~~chordPatt; + pitches=pitch|("["~~pitch..~~"]"); + note=preOperator~~pitches~~pitOperator~~volOperator~~durOperator~~postOperator; pitchTok=StringCases[ StringExpression[ sd:scaleDegree, po:pitOperator, - cn:chordNotation, - co:chordOperator + ch:chordPatt ]:><| "ScaleDegree"->sd, "PitchOperators"->po, - "ChordNotations"->cn, - "ChordOperators"->co + "Chord"->ch |> ]; @@ -298,7 +292,7 @@ Tokenizer[filepath_]:=Block[ }][[1]]&}; (* function simplified *) - object=("{"~~subtrack~~"}")|(notationPatt...~~note~~notationPatt...); + object=("{"~~subtrack~~"}")|(notationPadded...~~note~~notationPadded...); typePatt=<|"$"->string,"%"->expression,"&"->object,"!"->number,"@"->subtrack|>; typeTok[type_,str_]:=Switch[type, "String",<|"Type"->"String","Content"->str|>, @@ -426,8 +420,8 @@ EndPackage[]; (* ::Input:: *) -(*Tokenizer[NotebookDirectory[]<>"test.sml"][["Tokenizer","Sections"]]*) +(*Tokenizer[NotebookDirectory[]<>"test.sml"][["Tokenizer","Sections",1,"Tracks",1,"Content"(*,All,"Type"*)]]*) (* ::Input:: *) -(*Export[NotebookDirectory[]<>"test4.json",Tokenizer[NotebookDirectory[]<>"test4.sml"][["Tokenizer"]]];//Timing*) +(*Export[NotebookDirectory[]<>"test5.json",Tokenizer[NotebookDirectory[]<>"test5.sml"][["Tokenizer"]]];//Timing*) diff --git a/Lib/parser.wl b/Lib/parser.wl index 029f35b..7a47315 100644 --- a/Lib/parser.wl +++ b/Lib/parser.wl @@ -375,7 +375,7 @@ parse[tokenizer_,sections_]:=Module[ (* ::Input:: *) (*AudioStop[];AudioPlay[#[[2]]]&@*) (*EchoFunction["time: ",#[[1]]&]@*) -(*Timing[integrate[#MusicClips,#Effects]&[parse[QYS`Tokenize[localPath<>"Songs\\Touhou\\Deep_Mountain.qys"],2]]];*) +(*Timing[integrate[#MusicClips,#Effects]&[parse[QYS`Tokenize[localPath<>"Songs\\Touhou\\Deep_Mountain.qys"],3]]];*) (* ::Input:: *) diff --git a/Lib/std/Standard.smp b/Lib/std/Standard.smp index 668ddf9..924f16f 100644 --- a/Lib/std/Standard.smp +++ b/Lib/std/Standard.smp @@ -4,15 +4,12 @@ # Include "./Global.smp" # Include "./Function.smp" -# ChordNotation +# Chord M Major 0, 4, 7 m Minor 0, 3, 7 a Augmented 0, 4, 8 d Diminished 0, 3, 6 - -# ChordOperator - t tres [], [1]+3 T tres+ [], [1]+4 q quattuor [], [1]+5 diff --git a/Songs/Touhou/Deep_Mountain.qys b/Songs/Touhou/Deep_Mountain.qys index 7df7e74..ac95938 100644 --- a/Songs/Touhou/Deep_Mountain.qys +++ b/Songs/Touhou/Deep_Mountain.qys @@ -87,7 +87,7 @@ 6_6=6_.66_5_6_|3-0_3_3_5_|6_6=6_.6(^7)1'_7_5_|3-0_3_3_5_\ 6_6=6_.66_5_6_|3-0_3_3_5_|6_6=6_.6(^7)1'_7_5_|6-0-| -<0.8>\ +\ 6_6=6_.66_5_6_|3-0_3_3_5_|6_6=6_.6(^7)1'_7_5_|3-0_3_3_5_\ 6_6=6_.66_5_6_|3-0_3_3_5_|6_6=6_.6(^7)1'_7_5_|6-0_1_1_2_\ 3_3=3_.33_2_3_|6,-0_1_1_2_|3_3=3_.3(^5)6_5_2_|1-0_1_1_2_\ diff --git a/Songs/Touhou/TH11-Chireiden/Hana_ni_Kaze.qys b/Songs/Touhou/TH11-Chireiden/Hana_ni_Kaze.qys index 2bf9d99..6cbd03f 100644 --- a/Songs/Touhou/TH11-Chireiden/Hana_ni_Kaze.qys +++ b/Songs/Touhou/TH11-Chireiden/Hana_ni_Kaze.qys @@ -135,7 +135,7 @@ 4.%.%-%%%|5.%.%-%%%|6.%.%-%%%|6.%.%-%%-\ 4.%.%-%%%|5.%.%-%%%|5#.%.%-%%%|6.%.%-%%%\ 4.%.%-%%%|5.%.%-%%%|6.%.%-%%%|6.%.%-%%-\ -{<0.4>[4o,6],13[14]----|5o,,7,25-257|5#o,,7,2[35#]----|6o,,136-316,\ +{<0.4>[4o6,],13[14]----|5o,,7,25-257|5#o,,7,2[35#]----|6o,,136-316,\ 4o,,6,1[36]----|5o,,7,25-27,5,_[6,1]_|[4,,46]-^|%-}| <0.5>\ @@ -355,7 +355,7 @@ 4.%.%-%%%|5.%.%-%%%|5#.%.%-%%%|6.%.%-%%%\ 4.%.%-%%%|5.%.%-%%%|6.%.%-%%%|6.%.%-%%-\ }\ -{<0.4>[4o,6],13[14]----|5o,,7,25-257|5#o,,7,2[35#]----|6o,,136-316,\ +{<0.4>[4,o6],13[14]----|5o,,7,25-257|5#o,,7,2[35#]----|6o,,136-316,\ 4o,,6,1[36]----|5o,,7,25-27,5,|[6,1]-^|%-}| <0.5>{2*\ From 51115e7de884c6971dd7e137f21204ca35311095 Mon Sep 17 00:00:00 2001 From: Kouchya <312533284@qq.com> Date: Thu, 15 Feb 2018 21:10:37 +0800 Subject: [PATCH 2/3] Fixed an error in Taeta Hashi. --- Songs/Touhou/TH11-Chireiden/Taeta_Hashi.qys | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Songs/Touhou/TH11-Chireiden/Taeta_Hashi.qys b/Songs/Touhou/TH11-Chireiden/Taeta_Hashi.qys index dbaa4f4..72d49ea 100644 --- a/Songs/Touhou/TH11-Chireiden/Taeta_Hashi.qys +++ b/Songs/Touhou/TH11-Chireiden/Taeta_Hashi.qys @@ -108,9 +108,9 @@ xxxx|xxxx|xxxx|xxxx|| 0---|0---|0---|0---\ 0---|0---|0---|0---\ 0---|0---|0---|0---\ -0---|0---|4#o`_[61'#]o`_5#o`_[61'#]o`_4#o`_[61'#]o`__4#o`__4#o`_[61'#]o`_|4#o`_[61'#]o`_5#o`_[61'#]o`_4#o`_[61'#]o`__4#o`__4#o`_[61'#]o`_\ -4#o`_[61'#]o`_5#o`_[61'#]o`_4#o`_[61'#]o`__4#o`__4#o`_[61'#]o`_|4#o`_[61'#]o`_5#o`_[61'#]o`_4#o`_[61'#]o`__4#o`__4#o`_[61'#]o`_\ -<1=C,>4#o`_[61'#]o`_5#o`_[61'#]o`_4#o`_[61'#]o`__4#o`__4#o`_[61'#]o`_|4#o`_[61'#]o`_5#o`_[61'#]o`_4#o`_[61'#]o`__4#o`__4#o`_[61'#]o`_\ +0---|0---|4#o`_[61'#]`_5#o`_[61'#]`_4#o`_[61'#]`__4#o`__4#o`_[61'#]`_|4#o`_[61'#]`_5#o`_[61'#]`_4#o`_[61'#]`__4#o`__4#o`_[61'#]`_\ +4#o`_[61'#]`_5#o`_[61'#]`_4#o`_[61'#]`__4#o`__4#o`_[61'#]`_|4#o`_[61'#]`_5#o`_[61'#]`_4#o`_[61'#]`__4#o`__4#o`_[61'#]`_\ +<1=C,>4#o`_[61'#]`_5#o`_[61'#]`_4#o`_[61'#]`__4#o`__4#o`_[61'#]`_|4#o`_[61'#]`_5#o`_[61'#]`_4#o`_[61'#]`__4#o`__4#o`_[61'#]`_\ <1=C>4#po`_6po`_5#po`_6po`_4#po`_6po`__4#po`__4#po`_6po`_|4#po`_6po`_5#po`_6po`_4#po-\ 4,#o---|[35#]---|[26]--[261'#]|1#M--[5#6]\ [37]-3o-|[24#]'-[27]2o|[1#3]'---|[74'#]-[15]'#-\ From 2bbb1c1de534209d708ac3353f3833c4a1cb7c00 Mon Sep 17 00:00:00 2001 From: Shigma <1700011071@pku.edu.cn> Date: Thu, 15 Feb 2018 21:54:59 +0800 Subject: [PATCH 3/3] Add some adjustments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. 删除了Storage标签 2. 将和弦符号与和弦操作符整合成Chord 3. 修复了倚音不能正常tok的bug --- Lib/Tokenizer.wl | 26 ++++++++------------------ 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/Lib/Tokenizer.wl b/Lib/Tokenizer.wl index 7e6cb9a..206554e 100644 --- a/Lib/Tokenizer.wl +++ b/Lib/Tokenizer.wl @@ -1,7 +1,7 @@ (* ::Package:: *) (* ::Input:: *) -(*StringMatchQ[",",RegularExpression["\\,"]]*) +(*StringMatchQ["&",RegularExpression["[^\\{\\}]*"]]*) BeginPackage["SMML`Tokenizer`"]; @@ -11,13 +11,12 @@ rep[pat_]:=rep[pat,","~~" "...]; rep[pat_,sep_]:=pat~~(sep~~pat)...; repRegex[pat_]:=repRegex[pat,", *"]; repRegex[pat_,sep_]:=RE[pat<>"("<>sep<>pat<>")*"]; -sel[pat_RegularExpression]:=RE["("<>pat[[1]]<>")?"]; +sel[pat_]:=RE["("<>pat[[1]]<>")?"]; +or[pat__]:=RE["("<>StringRiffle[#[[1]]&/@{pat},"|"]<>")"]; unsigned=RE["\\d+"]; signed=RE["[\\+\\-]\\d+"]; integer=RE["[\\+\\-]?\\d+"]; -chordCodePos=RE["([\\+\\-]?\\d+)?"]; -chordCodeSft=RE["([\\+\\-]\\d+)?"]; chordCodeTok[str_]:=StringCases[str, StringExpression[ ntt:LetterCharacter~~"\t".., @@ -66,7 +65,7 @@ postOperator="``"|"`"|""; volOperator=(">"|":")...; pitOperator=("#"|"b"|"'"|",")...; durOperator=("."|"-"|"_"|"=")...; -scaleDegree="0"|"1"|"2"|"3"|"4"|"5"|"6"|"7"|"%"|"x"; +scaleDegree=RE["[0-7%x]"]; number=integer~~""|("."~~unsigned); expression=(integer~~""|("/"|"."~~unsigned))|("Log2("~~unsigned~~")"); @@ -81,7 +80,7 @@ orderTok=Union@@StringCases[#,{ n:integer~~"~"~~m:integer:>Range[ToExpression@n,ToExpression@m], n:integer:>{ToExpression@n} }]&; -notationPadded=Whitespace|"|"|"/"|"^"|"&"|"*"; +notationPadded=RE["[&\\|\\s\\^\\*]*"]; notationPatt=Alternatives[ "+"|"ToCoda"|"Coda"|"s"|"Segno"|"DC"|"DaCapo"|"DS"|"DaSegno", "||:"|":||"|("["~~orderListP~~".]"), @@ -166,7 +165,7 @@ Tokenizer[filepath_]:=Block[ ][line]; If[contextData!={}, syntax[[context]]=Union[syntax[[context]],contextData[[All,syntaxTags[[context]]]]]; - AppendTo[library,<|"Type"->context,"Storage"->"Internal","Data"->contextData|>]; + AppendTo[library,<|"Type"->context,"Data"->contextData|>]; contextData={}; ]; Switch[command, @@ -187,7 +186,6 @@ Tokenizer[filepath_]:=Block[ {item,{"Chord","Function"}}]; AppendTo[library,<| "Type"->"Package", - "Storage"->"External", "Path"->StringTake[value,{2,-2}], "Content"->source[["Tokenizer","Library"]] |>], @@ -292,8 +290,8 @@ Tokenizer[filepath_]:=Block[ }][[1]]&}; (* function simplified *) - object=("{"~~subtrack~~"}")|(notationPadded...~~note~~notationPadded...); - typePatt=<|"$"->string,"%"->expression,"&"->object,"!"->number,"@"->subtrack|>; + object=("{"~~subtrack~~"}")|(notationPadded~~note~~notationPadded); + typePatt=<|"$"->string,"%"->expression,"&"->object,"!"->number,"@"->Shortest[subtrack]|>; typeTok[type_,str_]:=Switch[type, "String",<|"Type"->"String","Content"->str|>, "Expression",<|"Type"->"Expression","Content"->str|>, @@ -411,14 +409,6 @@ Tokenizer[filepath_]:=Block[ EndPackage[]; -(* ::Input:: *) -(*Contexts["SMML`*"]*) - - -(* ::Input:: *) -(*Export[NotebookDirectory[]<>"test.json",%];*) - - (* ::Input:: *) (*Tokenizer[NotebookDirectory[]<>"test.sml"][["Tokenizer","Sections",1,"Tracks",1,"Content"(*,All,"Type"*)]]*)