From 2cf0149a3c854ac3ae9e54c4c786474b55161889 Mon Sep 17 00:00:00 2001 From: Tristan Hamilton Date: Mon, 2 Feb 2015 14:50:22 +0100 Subject: [PATCH 01/31] 'cherry picked' files from Ali's InfoRequestHandler-Code-Refactored branch which were relevant to helping filter out the unit selection package. Also formatted the code to eclipse style and added the google gson to pom --- marytts-runtime/pom.xml | 4 + .../java/marytts/modules/synthesis/Voice.java | 141 +++++- .../server/http/InfoRequestHandler.java | 365 +++++++++----- .../java/marytts/util/MaryRuntimeUtils.java | 83 +++- .../documentation/MaryDocumentationTest.java | 470 ++++++++++++++++++ pom.xml | 8 +- 6 files changed, 895 insertions(+), 176 deletions(-) create mode 100644 marytts-runtime/src/test/java/marytts/documentation/MaryDocumentationTest.java diff --git a/marytts-runtime/pom.xml b/marytts-runtime/pom.xml index c372d3fe86..fc21e003ff 100644 --- a/marytts-runtime/pom.xml +++ b/marytts-runtime/pom.xml @@ -65,6 +65,10 @@ freetts-en_us + + com.google.code.gson + gson + com.google.guava guava diff --git a/marytts-runtime/src/main/java/marytts/modules/synthesis/Voice.java b/marytts-runtime/src/main/java/marytts/modules/synthesis/Voice.java index 8607e473a4..900b586930 100644 --- a/marytts-runtime/src/main/java/marytts/modules/synthesis/Voice.java +++ b/marytts-runtime/src/main/java/marytts/modules/synthesis/Voice.java @@ -42,6 +42,7 @@ import javax.sound.sampled.AudioFormat; import javax.sound.sampled.AudioInputStream; +import javax.swing.Box.Filler; import marytts.cart.DirectedGraph; import marytts.cart.io.DirectedGraphReader; @@ -55,6 +56,7 @@ import marytts.exceptions.SynthesisException; import marytts.features.FeatureProcessorManager; import marytts.features.FeatureRegistry; +import marytts.htsengine.HMMVoice; import marytts.modules.MaryModule; import marytts.modules.ModuleRegistry; import marytts.modules.acoustic.BoundaryModel; @@ -66,6 +68,7 @@ import marytts.modules.phonemiser.Allophone; import marytts.modules.phonemiser.AllophoneSet; import marytts.server.MaryProperties; +import marytts.unitselection.UnitSelectionVoice; import marytts.unitselection.data.FeatureFileReader; import marytts.unitselection.interpolation.InterpolatingSynthesizer; import marytts.unitselection.interpolation.InterpolatingVoice; @@ -135,6 +138,8 @@ public int compare(Voice v1, Voice v2) { protected static Logger logger = MaryUtils.getLogger("Voice"); + private static final String FILENAME = "Voice.java:"; + /** A local map of already-instantiated Lexicons */ private static Map lexicons = new HashMap(); @@ -167,7 +172,8 @@ public Voice(String name, Locale locale, AudioFormat dbAudioFormat, WaveformSynt try { init(); } catch (Exception n) { - throw new MaryConfigurationException("Cannot instantiate voice '" + voiceName + "'", n); + throw new MaryConfigurationException(FILENAME + " Cannot instantiate voice '" + voiceName + "'" + "\tCause: " + + (n.getCause() != null ? n.getCause().getMessage() : "Cause is null!"), n); } } @@ -176,7 +182,8 @@ public Voice(String name, WaveformSynthesizer synthesizer) throws MaryConfigurat this.synthesizer = synthesizer; VoiceConfig config = MaryConfig.getVoiceConfig(voiceName); if (config == null) { - throw new MaryConfigurationException("Trying to load config for voice '" + voiceName + "' but cannot find it."); + throw new MaryConfigurationException(FILENAME + " Trying to load config for voice '" + voiceName + + "' but cannot find it."); } this.locale = config.getLocale(); int samplingRate = MaryProperties.getInteger("voice." + voiceName + ".samplingRate", 16000); @@ -192,7 +199,8 @@ public Voice(String name, WaveformSynthesizer synthesizer) throws MaryConfigurat try { init(); } catch (Exception n) { - throw new MaryConfigurationException("Cannot instantiate voice '" + voiceName + "'", n); + throw new MaryConfigurationException(FILENAME + " Cannot instantiate voice '" + voiceName + "'\tCause: " + + (n.getCause() != null ? n.getCause().getMessage() : "Cause is null!"), n); } } @@ -211,8 +219,9 @@ private void init() throws MaryConfigurationException, NoSuchPropertyException, try { allophoneSet = MaryRuntimeUtils.needAllophoneSet(MaryProperties.localePrefix(getLocale()) + ".allophoneset"); } catch (MaryConfigurationException e2) { - throw new MaryConfigurationException("No allophone set specified -- neither for voice '" + getName() - + "' nor for locale '" + getLocale() + "'", e2); + throw new MaryConfigurationException(FILENAME + " No allophone set specified -- neither for voice '" + getName() + + "' nor for locale '" + getLocale() + "'" + "\tCause: " + + (e2.getCause() != null ? e2.getCause().getMessage() : "Cause is null!"), e2); } } preferredModulesClasses = MaryProperties.getProperty(header + ".preferredModules"); @@ -241,7 +250,8 @@ private void loadOldStyleProsodyModels(String header) throws MaryConfigurationEx try { durationGraph = (new DirectedGraphReader()).load(durationGraphFile); } catch (IOException e) { - throw new MaryConfigurationException("Cannot load duration graph file '" + durationGraphFile + "'", e); + throw new MaryConfigurationException(FILENAME + " Cannot load duration graph file '" + durationGraphFile + "'" + + "\tCause: " + (e.getCause() != null ? e.getCause().getMessage() : "Cause is null!"), e); } } @@ -255,7 +265,8 @@ private void loadOldStyleProsodyModels(String header) throws MaryConfigurationEx String f0ContourFile = MaryProperties.needFilename(header + ".f0.contours"); f0ContourFeatures = new FeatureFileReader(f0ContourFile); } catch (IOException e) { - throw new MaryConfigurationException("Cannot load f0 contour graph file '" + f0GraphFile + "'", e); + throw new MaryConfigurationException(FILENAME + " Cannot load f0 contour graph file '" + f0GraphFile + "'" + + "\tCause: " + (e.getCause() != null ? e.getCause().getMessage() : "Cause is null!"), e); } } } @@ -301,7 +312,7 @@ private void loadAcousticModels(String header) throws MaryConfigurationException ModelType possibleModelTypes = ModelType.fromString(modelType); // if modelType is not in ModelType.values(), we don't know how to handle it: if (possibleModelTypes == null) { - throw new MaryConfigurationException("Cannot handle unknown model type: " + modelType); + throw new MaryConfigurationException(FILENAME + " Cannot handle unknown model type: " + modelType); } // ...and instantiate it in a switch statement: @@ -337,8 +348,9 @@ private void loadAcousticModels(String header) throws MaryConfigurationException break; } } catch (Throwable t) { - throw new MaryConfigurationException("Cannot instantiate model '" + modelName + "' of type '" + modelType - + "' from '" + MaryProperties.getProperty(header + "." + modelName + ".data") + "'", t); + throw new MaryConfigurationException(FILENAME + " Cannot instantiate model '" + modelName + "' of type '" + + modelType + "' from '" + MaryProperties.getProperty(header + "." + modelName + ".data") + "'" + + "\tCause: " + (t.getCause() != null ? t.getCause().getMessage() : "Cause is null!"), t); } // if we got this far, model should not be null: @@ -368,8 +380,9 @@ private void initFeatureProcessorManager() throws MaryConfigurationException { try { featMgr = (FeatureProcessorManager) Class.forName(featMgrClass).newInstance(); } catch (Exception e) { - throw new MaryConfigurationException("Cannot initialise voice-specific FeatureProcessorManager " + featMgrClass - + " from config file", e); + throw new MaryConfigurationException(FILENAME + " Cannot initialise voice-specific FeatureProcessorManager " + + featMgrClass + " from config file" + "\tCause: " + + (e.getCause() != null ? e.getCause().getMessage() : "Cause is null!"), e); } } else if (getOtherModels() != null) { // Only if there is no feature manager setting in the config file, @@ -383,10 +396,13 @@ private void initFeatureProcessorManager() throws MaryConfigurationException { Constructor fpmVoiceConstructor = fpmClass.getConstructor(Voice.class); featMgr = fpmVoiceConstructor.newInstance(this); } catch (NoSuchMethodException nsme) { - throw new MaryConfigurationException("Cannot initialise voice-specific FeatureProcessorManager: Class " - + fpmClass.getName() + " has no constructor " + fpmClass.getSimpleName() + "(Voice)"); + throw new MaryConfigurationException(FILENAME + + " Cannot initialise voice-specific FeatureProcessorManager: Class " + fpmClass.getName() + + " has no constructor " + fpmClass.getSimpleName() + "(Voice)" + "\tCause: " + + (nsme.getCause() != null ? nsme.getCause().getMessage() : "Cause is null!")); } catch (Exception e) { - throw new MaryConfigurationException("Cannot initialise voice-specific FeatureProcessorManager", e); + throw new MaryConfigurationException(FILENAME + " Cannot initialise voice-specific FeatureProcessorManager" + + "\tCause: " + (e.getCause() != null ? e.getCause().getMessage() : "Cause is null!"), e); } } // register the FeatureProcessorManager for this Voice: @@ -408,10 +424,8 @@ public AllophoneSet getAllophoneSet() { * Get the Allophone set for the given phone symbol. * * @param phoneSymbol - * @return an Allophone object if phoneSymbol is a known phone symbol in the voice's AllophoneSet. - * @deprecated use {@link AllophoneSet#getAllophone(String)} directly instead + * @return an Allophone object if phoneSymbol is a known phone symbol in the voice's AllophoneSet, or null. */ - @Deprecated public Allophone getAllophone(String phoneSymbol) { return allophoneSet.getAllophone(phoneSymbol); } @@ -463,6 +477,10 @@ public boolean hasName(String aName) { return voiceName.equals(aName); } + public boolean hasNameIgnoreCase(String aName) { + return voiceName.equalsIgnoreCase(aName); + } + /** * Return the name of this voice. If the voice has several possible names, the first one is returned. */ @@ -621,7 +639,7 @@ public Map getOtherModels() { */ public static void registerVoice(Voice voice) { if (voice == null) - throw new NullPointerException("Cannot register null voice."); + throw new NullPointerException(FILENAME + " Cannot register null voice."); if (!allVoices.contains(voice)) { logger.info("Registering voice `" + voice.getName() + "': " + voice.gender() + ", locale " + voice.getLocale()); allVoices.add(voice); @@ -691,9 +709,81 @@ public static Voice getVoice(String name) { * their "wantToBeDefault" value. */ public static Collection getAvailableVoices() { + // if (allVoices == null || allVoices.size() == 0) + // { + // fillVoices(); + // } return Collections.unmodifiableSet(allVoices); } + public static List getFilteredVoice(String queryVariable, String queryValue, Collection initialList) { + List filteredList = new ArrayList(); + + switch (queryVariable.toLowerCase()) { + case "locale": + Locale queryLocale = MaryUtils.string2locale(queryValue); + for (Voice currentVoice : initialList) { + if (MaryUtils.subsumes(queryLocale, currentVoice.getLocale())) { + filteredList.add(currentVoice); + } + } + break; + + case "name": + for (Voice currentVoice : initialList) { + if (currentVoice.hasNameIgnoreCase(queryValue)) { + filteredList.add(currentVoice); + } + } + break; + + case "gender": + for (Voice currentVoice : initialList) { + if (currentVoice.gender().name.equalsIgnoreCase(queryValue)) { + filteredList.add(currentVoice); + } + } + break; + + case "type": + for (Voice currentVoice : initialList) { + boolean isQualified = false; + if (queryValue.toLowerCase().contains("unit") && currentVoice instanceof UnitSelectionVoice) { + isQualified = true; + } else if (queryValue.toLowerCase().contains("hmm") && currentVoice instanceof HMMVoice) { + isQualified = true; + } + + if (isQualified) { + filteredList.add(currentVoice); + } + } + break; + + default: + break; + } + + return filteredList; + } + + public static Collection getAvailableVoices(final Map queryParameters) { + List filteredVoices = new ArrayList(); + + int index = 0; + for (String key : queryParameters.keySet()) { + if (index == 0) { + filteredVoices = getFilteredVoice(key, queryParameters.get(key), allVoices); + } else { + filteredVoices = getFilteredVoice(key, queryParameters.get(key), filteredVoices); + } + + index++; + } + + return filteredVoices; + } + /** * Get the list of all available voices for a given locale. The iterator of the collection returned will return the voices in * decreasing order of their "wantToBeDefault" value. @@ -719,7 +809,7 @@ public static Collection getAvailableVoices(Locale locale) { */ public static Collection getAvailableVoices(WaveformSynthesizer synth) { if (synth == null) { - throw new NullPointerException("Got null WaveformSynthesizer"); + throw new NullPointerException(FILENAME + " Got null WaveformSynthesizer"); } ArrayList list = new ArrayList(); for (Voice v : allVoices) { @@ -787,7 +877,7 @@ public static Voice getDefaultVoice(Locale locale) { if (v == null) v = getVoice(locale, MALE); if (v == null) - logger.debug("Could not find default voice for locale " + locale); + logger.warn("Could not find default voice for locale " + locale); return v; } @@ -890,4 +980,11 @@ public boolean equals(Gender other) { } } -} + public void setVoiceName(String voiceName) { + this.voiceName = voiceName; + } + + public void setLocale(Locale locale) { + this.locale = locale; + } +} \ No newline at end of file diff --git a/marytts-runtime/src/main/java/marytts/server/http/InfoRequestHandler.java b/marytts-runtime/src/main/java/marytts/server/http/InfoRequestHandler.java index 26bf65f161..765d7f717b 100644 --- a/marytts-runtime/src/main/java/marytts/server/http/InfoRequestHandler.java +++ b/marytts-runtime/src/main/java/marytts/server/http/InfoRequestHandler.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.io.UnsupportedEncodingException; +import java.util.HashMap; import java.util.Locale; import java.util.Map; @@ -41,9 +42,230 @@ * @author Oytun Türk, Marc Schröder */ public class InfoRequestHandler extends BaseHttpRequestHandler { + // CorrectMethod interface and HashMap methodsMapping serve the purpose of directing URL to correct function + public interface CorrectMethod { + String callCorrectMethod(Map queryItems, HttpResponse response); + } + + public Map methodsMapping; public InfoRequestHandler() { super(); + methodsMapping = new HashMap(); + + methodsMapping.put("voices", new CorrectMethod() { + @Override + public String callCorrectMethod(Map queryItems, HttpResponse response) { + return MaryRuntimeUtils.getVoices(queryItems); + } + }); + + methodsMapping.put("version", new CorrectMethod() { + @Override + public String callCorrectMethod(Map queryItems, HttpResponse response) { + return MaryRuntimeUtils.getMaryVersion(); + } + }); + + methodsMapping.put("datatypes", new CorrectMethod() { + @Override + public String callCorrectMethod(Map queryItems, HttpResponse response) { + return MaryRuntimeUtils.getDataTypes(); + } + }); + + methodsMapping.put("locales", new CorrectMethod() { + @Override + public String callCorrectMethod(Map queryItems, HttpResponse response) { + return MaryRuntimeUtils.getLocales(); + } + }); + + methodsMapping.put("audioformats", new CorrectMethod() { + @Override + public String callCorrectMethod(Map queryItems, HttpResponse response) { + return MaryRuntimeUtils.getAudioFileFormatTypes(); + } + }); + + methodsMapping.put("exampletext", new CorrectMethod() { + @Override + public String callCorrectMethod(Map queryItems, HttpResponse response) { + if (queryItems != null) { + // Voice example text + String voice = queryItems.get("voice"); + if (voice != null) { + return MaryRuntimeUtils.getVoiceExampleText(voice); + } + String datatype = queryItems.get("datatype"); + String locale = queryItems.get("locale"); + if (datatype != null && locale != null) { + Locale loc = MaryUtils.string2locale(locale); + return MaryRuntimeUtils.getExampleText(datatype, loc); + } + } + MaryHttpServerUtils.errorMissingQueryParameter(response, "'datatype' and 'locale' or 'voice'"); + return null; + } + }); + + methodsMapping.put("audioeffects", new CorrectMethod() { + @Override + public String callCorrectMethod(Map queryItems, HttpResponse response) { + return MaryRuntimeUtils.getDefaultAudioEffects(); + } + }); + + methodsMapping.put("audioeffect-default-param", new CorrectMethod() { + @Override + public String callCorrectMethod(Map queryItems, HttpResponse response) { + if (queryItems != null) { + String effect = queryItems.get("effect"); + if (effect != null) + return MaryRuntimeUtils.getAudioEffectDefaultParam(effect); + } + MaryHttpServerUtils.errorMissingQueryParameter(response, "'effect'"); + return null; + } + }); + + methodsMapping.put("audioeffect-full", new CorrectMethod() { + @Override + public String callCorrectMethod(Map queryItems, HttpResponse response) { + if (queryItems != null) { + String effect = queryItems.get("effect"); + String params = queryItems.get("params"); + if (effect != null && params != null) { + return MaryRuntimeUtils.getFullAudioEffect(effect, params); + } + } + MaryHttpServerUtils.errorMissingQueryParameter(response, "'effect' and 'params'"); + return null; + } + }); + + methodsMapping.put("audioeffect-help", new CorrectMethod() { + @Override + public String callCorrectMethod(Map queryItems, HttpResponse response) { + if (queryItems != null) { + String effect = queryItems.get("effect"); + if (effect != null) { + return MaryRuntimeUtils.getAudioEffectHelpText(effect); + } + } + MaryHttpServerUtils.errorMissingQueryParameter(response, "'effect'"); + return null; + } + }); + + methodsMapping.put("audioeffect-is-hmm-effect", new CorrectMethod() { + @Override + public String callCorrectMethod(Map queryItems, HttpResponse response) { + if (queryItems != null) { + String effect = queryItems.get("effect"); + if (effect != null) { + return MaryRuntimeUtils.isHmmAudioEffect(effect); + } + } + MaryHttpServerUtils.errorMissingQueryParameter(response, "'effect'"); + return null; + } + }); + + methodsMapping.put("features", new CorrectMethod() { + @Override + public String callCorrectMethod(Map queryItems, HttpResponse response) { + return getRequestedFeature(false, queryItems, response); + } + }); + + methodsMapping.put("features-discrete", new CorrectMethod() { + @Override + public String callCorrectMethod(Map queryItems, HttpResponse response) { + return getRequestedFeature(true, queryItems, response); + } + }); + + methodsMapping.put("vocalizations", new CorrectMethod() { + @Override + public String callCorrectMethod(Map queryItems, HttpResponse response) { + if (queryItems != null) { + String voice = queryItems.get("voice"); + if (voice != null) { + return MaryRuntimeUtils.getVocalizations(voice); + } + } + MaryHttpServerUtils.errorMissingQueryParameter(response, "'voice'"); + return null; + } + }); + + methodsMapping.put("styles", new CorrectMethod() { + @Override + public String callCorrectMethod(Map queryItems, HttpResponse response) { + if (queryItems != null) { + String voice = queryItems.get("voice"); + if (voice != null) { + return MaryRuntimeUtils.getStyles(voice); + } + } + MaryHttpServerUtils.errorMissingQueryParameter(response, "'voice'"); + return null; + } + }); + } + + protected String getRequestedFeature(boolean isFeatureDiscreteRequest, Map queryItems, HttpResponse response) { + if (queryItems != null) { + // List of features that can be computed for the voice + FeatureProcessorManager mgr = null; + String voiceName = queryItems.get("voice"); + String localeName = queryItems.get("locale"); + if (voiceName != null) { + Voice voice = Voice.getVoice(voiceName); + if (voice == null) { + MaryHttpServerUtils.errorWrongQueryParameterValue(response, "voice", voiceName, "No voice with that name"); + return null; + } + mgr = FeatureRegistry.getFeatureProcessorManager(voice); + if (mgr == null) { + mgr = FeatureRegistry.getFeatureProcessorManager(voice.getLocale()); + } + if (mgr == null) { + mgr = FeatureRegistry.getFeatureProcessorManager(new Locale(voice.getLocale().getLanguage())); + } + if (mgr == null) { + mgr = FeatureRegistry.getFallbackFeatureProcessorManager(); + } + } else if (localeName != null) { + Locale locale = MaryUtils.string2locale(localeName); + mgr = FeatureRegistry.getFeatureProcessorManager(locale); + if (mgr == null) { + mgr = FeatureRegistry.getFeatureProcessorManager(new Locale(locale.getLanguage())); + } + if (mgr == null) { + StringBuilder localeList = new StringBuilder(); + for (Locale l : FeatureRegistry.getSupportedLocales()) { + if (localeList.length() > 0) + localeList.append(","); + localeList.append(l.toString()); + } + MaryHttpServerUtils.errorWrongQueryParameterValue(response, "locale", localeName, + "The locale is not supported.
" + "Supported locales: " + localeList + ""); + return null; + } + } + if (mgr != null) + // if (request.equals("features-discrete")) { + if (isFeatureDiscreteRequest) { + String discreteFeatureNames = mgr.listByteValuedFeatureProcessorNames() + + mgr.listShortValuedFeatureProcessorNames(); + return discreteFeatureNames; + } + return mgr.listFeatureProcessorNames(); + } + MaryHttpServerUtils.errorMissingQueryParameter(response, "'voice' or 'locale'"); + return null; } @Override @@ -75,142 +297,13 @@ private String handleInfoRequest(String absPath, Map queryItems, assert absPath.startsWith("/") : "Absolute path '" + absPath + "' does not start with a slash!"; String request = absPath.substring(1); // without the initial slash - if (request.equals("version")) - return MaryRuntimeUtils.getMaryVersion(); - else if (request.equals("datatypes")) - return MaryRuntimeUtils.getDataTypes(); - else if (request.equals("locales")) - return MaryRuntimeUtils.getLocales(); - else if (request.equals("voices")) - return MaryRuntimeUtils.getVoices(); - else if (request.equals("audioformats")) - return MaryRuntimeUtils.getAudioFileFormatTypes(); - else if (request.equals("exampletext")) { - if (queryItems != null) { - // Voice example text - String voice = queryItems.get("voice"); - if (voice != null) { - return MaryRuntimeUtils.getVoiceExampleText(voice); - } - String datatype = queryItems.get("datatype"); - String locale = queryItems.get("locale"); - if (datatype != null && locale != null) { - Locale loc = MaryUtils.string2locale(locale); - return MaryRuntimeUtils.getExampleText(datatype, loc); - } - } - MaryHttpServerUtils.errorMissingQueryParameter(response, "'datatype' and 'locale' or 'voice'"); - return null; - } else if (request.equals("audioeffects")) - return MaryRuntimeUtils.getDefaultAudioEffects(); - else if (request.equals("audioeffect-default-param")) { - if (queryItems != null) { - String effect = queryItems.get("effect"); - if (effect != null) - return MaryRuntimeUtils.getAudioEffectDefaultParam(effect); - } - MaryHttpServerUtils.errorMissingQueryParameter(response, "'effect'"); - return null; - } else if (request.equals("audioeffect-full")) { - if (queryItems != null) { - String effect = queryItems.get("effect"); - String params = queryItems.get("params"); - if (effect != null && params != null) { - return MaryRuntimeUtils.getFullAudioEffect(effect, params); - } - } - MaryHttpServerUtils.errorMissingQueryParameter(response, "'effect' and 'params'"); - return null; - } else if (request.equals("audioeffect-help")) { - if (queryItems != null) { - String effect = queryItems.get("effect"); - if (effect != null) { - return MaryRuntimeUtils.getAudioEffectHelpText(effect); - } - } - MaryHttpServerUtils.errorMissingQueryParameter(response, "'effect'"); - return null; - } else if (request.equals("audioeffect-is-hmm-effect")) { - if (queryItems != null) { - String effect = queryItems.get("effect"); - if (effect != null) { - return MaryRuntimeUtils.isHmmAudioEffect(effect); - } - } - MaryHttpServerUtils.errorMissingQueryParameter(response, "'effect'"); - return null; - } else if (request.equals("features") || request.equals("features-discrete")) { - if (queryItems != null) { - // List of features that can be computed for the voice - FeatureProcessorManager mgr = null; - String voiceName = queryItems.get("voice"); - String localeName = queryItems.get("locale"); - if (voiceName != null) { - Voice voice = Voice.getVoice(voiceName); - if (voice == null) { - MaryHttpServerUtils - .errorWrongQueryParameterValue(response, "voice", voiceName, "No voice with that name"); - return null; - } - mgr = FeatureRegistry.getFeatureProcessorManager(voice); - if (mgr == null) { - mgr = FeatureRegistry.getFeatureProcessorManager(voice.getLocale()); - } - if (mgr == null) { - mgr = FeatureRegistry.getFeatureProcessorManager(new Locale(voice.getLocale().getLanguage())); - } - if (mgr == null) { - mgr = FeatureRegistry.getFallbackFeatureProcessorManager(); - } - } else if (localeName != null) { - Locale locale = MaryUtils.string2locale(localeName); - mgr = FeatureRegistry.getFeatureProcessorManager(locale); - if (mgr == null) { - mgr = FeatureRegistry.getFeatureProcessorManager(new Locale(locale.getLanguage())); - } - if (mgr == null) { - StringBuilder localeList = new StringBuilder(); - for (Locale l : FeatureRegistry.getSupportedLocales()) { - if (localeList.length() > 0) - localeList.append(","); - localeList.append(l.toString()); - } - MaryHttpServerUtils.errorWrongQueryParameterValue(response, "locale", localeName, - "The locale is not supported.
" + "Supported locales: " + localeList + ""); - return null; - } - } - if (mgr != null) - if (request.equals("features-discrete")) { - String discreteFeatureNames = mgr.listByteValuedFeatureProcessorNames() - + mgr.listShortValuedFeatureProcessorNames(); - return discreteFeatureNames; - } - return mgr.listFeatureProcessorNames(); - } - MaryHttpServerUtils.errorMissingQueryParameter(response, "'voice' or 'locale'"); - return null; - } else if (request.equals("vocalizations")) { - if (queryItems != null) { - String voice = queryItems.get("voice"); - if (voice != null) { - return MaryRuntimeUtils.getVocalizations(voice); - } - } - MaryHttpServerUtils.errorMissingQueryParameter(response, "'voice'"); - return null; - } else if (request.equals("styles")) { - if (queryItems != null) { - String voice = queryItems.get("voice"); - if (voice != null) { - return MaryRuntimeUtils.getStyles(voice); - } - } - MaryHttpServerUtils.errorMissingQueryParameter(response, "'voice'"); + String finalOutput = methodsMapping.get(request).callCorrectMethod(queryItems, response); + + if (finalOutput == null) { + MaryHttpServerUtils.errorFileNotFound(response, request); return null; + } else { + return finalOutput; } - MaryHttpServerUtils.errorFileNotFound(response, request); - return null; } - -} +} \ No newline at end of file diff --git a/marytts-runtime/src/main/java/marytts/util/MaryRuntimeUtils.java b/marytts-runtime/src/main/java/marytts/util/MaryRuntimeUtils.java index 9b3b4fd93d..644f5c0a7a 100644 --- a/marytts-runtime/src/main/java/marytts/util/MaryRuntimeUtils.java +++ b/marytts-runtime/src/main/java/marytts/util/MaryRuntimeUtils.java @@ -25,10 +25,15 @@ import java.io.InputStream; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; +import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Locale; +import java.util.Map; import java.util.StringTokenizer; import java.util.Vector; @@ -59,15 +64,21 @@ import org.w3c.dom.Element; +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; + /** * @author marc * */ public class MaryRuntimeUtils { + private static final String FILENAME = "MaryRuntimeUtils.java:"; public static void ensureMaryStarted() throws Exception { synchronized (MaryConfig.getMainConfig()) { + // System.out.println("in ensureMaryStarted()"); if (Mary.currentState() == Mary.STATE_OFF) { + System.out.println("Mary's current state is off. Starting it!"); Mary.startup(); } } @@ -131,8 +142,9 @@ public static Object instantiateObject(String objectInitInfo) throws MaryConfigu } } catch (Exception e) { // try to make e's message more informative if possible - throw new MaryConfigurationException("Cannot instantiate object from '" + objectInitInfo + "': " - + MaryUtils.getFirstMeaningfulMessage(e), e); + throw new MaryConfigurationException(FILENAME + "Cannot instantiate object from '" + objectInitInfo + "': " + + MaryUtils.getFirstMeaningfulMessage(e) + "\tCause: " + + (e.getCause() != null ? e.getCause().getMessage() : "Cause is null!"), e); } return obj; } @@ -311,7 +323,7 @@ else if (audiostoreProperty.equals("file")) public static AllophoneSet needAllophoneSet(String propertyName) throws MaryConfigurationException { String propertyValue = MaryProperties.getProperty(propertyName); if (propertyValue == null) { - throw new MaryConfigurationException("No such property: " + propertyName); + throw new MaryConfigurationException(FILENAME + " No such property: " + propertyName); } if (AllophoneSet.hasAllophoneSet(propertyValue)) { return AllophoneSet.getAllophoneSetById(propertyValue); @@ -320,7 +332,8 @@ public static AllophoneSet needAllophoneSet(String propertyName) throws MaryConf try { alloStream = MaryProperties.needStream(propertyName); } catch (FileNotFoundException e) { - throw new MaryConfigurationException("Cannot open allophone stream for property " + propertyName, e); + throw new MaryConfigurationException(FILENAME + " Cannot open allophone stream for property " + propertyName + + "\tCause: " + (e.getCause() != null ? e.getCause().getMessage() : "Cause is null!"), e); } assert alloStream != null; return AllophoneSet.getAllophoneSet(alloStream, propertyValue); @@ -360,31 +373,68 @@ public static String getLocales() { return out.toString(); } - public static String getVoices() { - String output = ""; - Collection voices = Voice.getAvailableVoices(); + public static String getVoices(Map queryParameters) { + String outputString = ""; + List voices; + + if (queryParameters == null || queryParameters.size() == 0) { + voices = new ArrayList(Voice.getAvailableVoices()); + } else { + voices = new ArrayList(Voice.getAvailableVoices(queryParameters)); + } + + sortVoicesList(voices); // sort voices by: locale + gender + name + type + outputString = formJSONFromVoices(voices); + return outputString; + } + + // /sort by: locale + gender + name + type + public static void sortVoicesList(List voices) { + Collections.sort(voices, new Comparator() { + @Override + public int compare(Voice v1, Voice v2) { + return (v1.getLocale().toString() + " " + v1.gender().toString() + " " + v1.getName() + " " + (v1 instanceof UnitSelectionVoice ? "u" + : "h")).compareToIgnoreCase(v2.getLocale().toString() + " " + v2.gender().toString() + " " + v2.getName() + + " " + (v2 instanceof UnitSelectionVoice ? "u" : "h")); + } + }); + } + + public static String formJSONFromVoices(Collection voices) { + JsonArray outputJson = new JsonArray(); + for (Iterator it = voices.iterator(); it.hasNext();) { + JsonObject currentVoice = new JsonObject(); Voice v = (Voice) it.next(); if (v instanceof InterpolatingVoice) { // do not list interpolating voice } else if (v instanceof UnitSelectionVoice) { - output += v.getName() + " " + v.getLocale() + " " + v.gender().toString() + " " + "unitselection" + " " - + ((UnitSelectionVoice) v).getDomain() + System.getProperty("line.separator"); + currentVoice.addProperty("name", v.getName()); + currentVoice.addProperty("locale", v.getLocale().toString()); + currentVoice.addProperty("gender", v.gender().toString()); + currentVoice.addProperty("type", "unitselection"); + currentVoice.addProperty("domain", ((UnitSelectionVoice) v).getDomain()); } else if (v instanceof HMMVoice) { - output += v.getName() + " " + v.getLocale() + " " + v.gender().toString() + " " + "hmm" - + System.getProperty("line.separator"); + currentVoice.addProperty("name", v.getName()); + currentVoice.addProperty("locale", v.getLocale().toString()); + currentVoice.addProperty("gender", v.gender().toString()); + currentVoice.addProperty("type", "hmm"); } else { - output += v.getName() + " " + v.getLocale() + " " + v.gender().toString() + " " + "other" - + System.getProperty("line.separator"); + currentVoice.addProperty("name", v.getName()); + currentVoice.addProperty("locale", v.getLocale().toString()); + currentVoice.addProperty("gender", v.gender().toString()); + currentVoice.addProperty("type", "other"); } + + outputJson.add(currentVoice); } - return output; + return outputJson.toString(); } public static String getDefaultVoiceName() { String defaultVoiceName = ""; - String allVoices = getVoices(); + String allVoices = getVoices(new HashMap()); if (allVoices != null && allVoices.length() > 0) { StringTokenizer tt = new StringTokenizer(allVoices, System.getProperty("line.separator")); if (tt.hasMoreTokens()) { @@ -510,5 +560,4 @@ public static String isHmmAudioEffect(String effectName) { } return effect.isHMMEffect() ? "yes" : "no"; } - -} +} \ No newline at end of file diff --git a/marytts-runtime/src/test/java/marytts/documentation/MaryDocumentationTest.java b/marytts-runtime/src/test/java/marytts/documentation/MaryDocumentationTest.java new file mode 100644 index 0000000000..520ad45e11 --- /dev/null +++ b/marytts-runtime/src/test/java/marytts/documentation/MaryDocumentationTest.java @@ -0,0 +1,470 @@ +package marytts.documentation; + +import static org.junit.Assert.assertEquals; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; + +import marytts.LocalMaryInterface; +import marytts.exceptions.MaryConfigurationException; +import marytts.modules.synthesis.Voice; +import marytts.server.http.InfoRequestHandler; + +import org.junit.Before; +import org.junit.Test; + +public class MaryDocumentationTest { + String voiceName = "cmu-slt-hsmm"; + + @Before + public void setUp() throws Exception { + try { + LocalMaryInterface mary = new LocalMaryInterface(); + // System.out.println("mary interface initialized"); + // HMMSynthesizer hmmSynth = new HMMSynthesizer(); + // System.out.println("hmm synthesizer initialized"); + // HMMVoice v = new HMMVoice(voiceName, hmmSynth); + // System.out.println("hmm voice initialized"); + // Voice.registerVoice(v); + } catch (Exception e) { + // e.printStackTrace(); + // System.out.println("ERROR: " + "setUp for MaryDocumentationTest failed.\tCause: " + e.getCause().getMessage()); + throw new Exception("setUp for MaryDocumentationTest failed.\tCause: " + + (e.getCause() != null ? e.getCause().getMessage() : "Cause is null!"), e); + } + } + + // @Test + // public void testAllVoices() { + // InfoRequestHandler requestHandler = new InfoRequestHandler(); + // + // String expectedOutput = "[{\"name\":\"cmu-slt-hsmm\",\"locale\":\"en_US\",\"gender\":\"female\",\"type\":\"hmm\"}]"; + // String actualOutput = requestHandler.methodsMapping.get("voices").callCorrectMethod(null, null); + // + // assertEquals(expectedOutput, actualOutput); + // } + + // @Test + // public void testVoicesWithLocale() { + // InfoRequestHandler requestHandler = new InfoRequestHandler(); + // Map queryVariables = new HashMap (); + // + // //expected output, locale=de: [] + // String expectedOutput = "[]"; + // queryVariables.put("locale", "de"); + // String actualOutput = requestHandler.methodsMapping.get("voices").callCorrectMethod(queryVariables, null); + // assertEquals(expectedOutput, actualOutput); + // + // //expected output, locale=en: [{"name":"cmu-slt-hsmm","locale":"en_US","gender":"female","type":"hmm"}] + // expectedOutput = "[{\"name\":\"cmu-slt-hsmm\",\"locale\":\"en_US\",\"gender\":\"female\",\"type\":\"hmm\"}]"; + // queryVariables.put("locale", "en"); + // actualOutput = requestHandler.methodsMapping.get("voices").callCorrectMethod(queryVariables, null); + // assertEquals(expectedOutput, actualOutput); + // + // //expected output, locale=en-us: [{"name":"cmu-slt-hsmm","locale":"en_US","gender":"female","type":"hmm"}] + // expectedOutput = "[{\"name\":\"cmu-slt-hsmm\",\"locale\":\"en_US\",\"gender\":\"female\",\"type\":\"hmm\"}]"; + // queryVariables.put("locale", "en-us"); + // actualOutput = requestHandler.methodsMapping.get("voices").callCorrectMethod(queryVariables, null); + // assertEquals(expectedOutput, actualOutput); + // + // //expected output, locale=en-gb: [] + // expectedOutput = "[]"; + // queryVariables.put("locale", "en-gb"); + // actualOutput = requestHandler.methodsMapping.get("voices").callCorrectMethod(queryVariables, null); + // assertEquals(expectedOutput, actualOutput); + // } + // + // @Test + // public void testVoicesWithName() { + // InfoRequestHandler requestHandler = new InfoRequestHandler(); + // Map queryVariables = new HashMap (); + // + // //expected output, name=cmu-slt-hsmm: [{"name":"cmu-slt-hsmm","locale":"en_US","gender":"female","type":"hmm"}] + // String expectedOutput = "[{\"name\":\"cmu-slt-hsmm\",\"locale\":\"en_US\",\"gender\":\"female\",\"type\":\"hmm\"}]"; + // queryVariables.put("name", "cmu-slt-hsmm"); + // String actualOutput = requestHandler.methodsMapping.get("voices").callCorrectMethod(queryVariables, null); + // assertEquals(expectedOutput, actualOutput); + // + // //expected output, name=dummy-name: [] + // expectedOutput = "[]"; + // queryVariables.put("name", "dummy-name"); + // actualOutput = requestHandler.methodsMapping.get("voices").callCorrectMethod(queryVariables, null); + // assertEquals(expectedOutput, actualOutput); + // } + // + // @Test + // public void testVoicesWithGender() { + // InfoRequestHandler requestHandler = new InfoRequestHandler(); + // Map queryVariables = new HashMap (); + // + // //expected output, gender=female: [{"name":"cmu-slt-hsmm","locale":"en_US","gender":"female","type":"hmm"}] + // String expectedOutput = "[{\"name\":\"cmu-slt-hsmm\",\"locale\":\"en_US\",\"gender\":\"female\",\"type\":\"hmm\"}]"; + // queryVariables.put("gender", "female"); + // String actualOutput = requestHandler.methodsMapping.get("voices").callCorrectMethod(queryVariables, null); + // assertEquals(expectedOutput, actualOutput); + // + // //expected output, gender=male: [] + // expectedOutput = "[]"; + // queryVariables.put("gender", "male"); + // actualOutput = requestHandler.methodsMapping.get("voices").callCorrectMethod(queryVariables, null); + // assertEquals(expectedOutput, actualOutput); + // } + // + // @Test + // public void testVoicesWithType() { + // InfoRequestHandler requestHandler = new InfoRequestHandler(); + // Map queryVariables = new HashMap (); + // + // //expected output, type=hmm: [{"name":"cmu-slt-hsmm","locale":"en_US","gender":"female","type":"hmm"}] + // String expectedOutput = "[{\"name\":\"cmu-slt-hsmm\",\"locale\":\"en_US\",\"gender\":\"female\",\"type\":\"hmm\"}]"; + // queryVariables.put("type", "hmm"); + // String actualOutput = requestHandler.methodsMapping.get("voices").callCorrectMethod(queryVariables, null); + // assertEquals(expectedOutput, actualOutput); + // + // //expected output, type=unit: [] + // expectedOutput = "[]"; + // queryVariables.put("type", "unit"); + // actualOutput = requestHandler.methodsMapping.get("voices").callCorrectMethod(queryVariables, null); + // assertEquals(expectedOutput, actualOutput); + // } + // + // @Test + // public void testVoicesWithLocaleAndName() { + // InfoRequestHandler requestHandler = new InfoRequestHandler(); + // Map queryVariables = new HashMap (); + // + // //expected output, locale=en-us, name=cmu-slt-hsmm: + // [{"name":"cmu-slt-hsmm","locale":"en_US","gender":"female","type":"hmm"}] + // String expectedOutput = "[{\"name\":\"cmu-slt-hsmm\",\"locale\":\"en_US\",\"gender\":\"female\",\"type\":\"hmm\"}]"; + // queryVariables.put("locale", "en-us"); + // queryVariables.put("name", "cmu-slt-hsmm"); + // String actualOutput = requestHandler.methodsMapping.get("voices").callCorrectMethod(queryVariables, null); + // assertEquals(expectedOutput, actualOutput); + // + // //expected output, locale=de, name=cmu-slt-hsmm: [] + // expectedOutput = "[]"; + // queryVariables.put("locale", "de"); + // queryVariables.put("name", "cmu-slt-hsmm"); + // actualOutput = requestHandler.methodsMapping.get("voices").callCorrectMethod(queryVariables, null); + // assertEquals(expectedOutput, actualOutput); + // + // //expected output, locale=en-us, name=dummy-name: [] + // expectedOutput = "[]"; + // queryVariables.put("locale", "en-us"); + // queryVariables.put("name", "dummy-name"); + // actualOutput = requestHandler.methodsMapping.get("voices").callCorrectMethod(queryVariables, null); + // assertEquals(expectedOutput, actualOutput); + // + // //expected output, locale=de, name=dummy-name: [] + // expectedOutput = "[]"; + // queryVariables.put("locale", "de"); + // queryVariables.put("name", "dummy-name"); + // actualOutput = requestHandler.methodsMapping.get("voices").callCorrectMethod(queryVariables, null); + // assertEquals(expectedOutput, actualOutput); + // } + // + // @Test + // public void testVoicesWithLocaleAndGender() { + // InfoRequestHandler requestHandler = new InfoRequestHandler(); + // Map queryVariables = new HashMap (); + // + // //expected output, locale=en-us, gender=female: [{"name":"cmu-slt-hsmm","locale":"en_US","gender":"female","type":"hmm"}] + // String expectedOutput = "[{\"name\":\"cmu-slt-hsmm\",\"locale\":\"en_US\",\"gender\":\"female\",\"type\":\"hmm\"}]"; + // queryVariables.put("locale", "en-us"); + // queryVariables.put("gender", "female"); + // String actualOutput = requestHandler.methodsMapping.get("voices").callCorrectMethod(queryVariables, null); + // assertEquals(expectedOutput, actualOutput); + // + // //expected output, locale=de, gender=female: [] + // expectedOutput = "[]"; + // queryVariables.put("locale", "de"); + // queryVariables.put("gender", "female"); + // actualOutput = requestHandler.methodsMapping.get("voices").callCorrectMethod(queryVariables, null); + // assertEquals(expectedOutput, actualOutput); + // + // //expected output, locale=en-us, gender=male: [] + // expectedOutput = "[]"; + // queryVariables.put("locale", "en-us"); + // queryVariables.put("gender", "male"); + // actualOutput = requestHandler.methodsMapping.get("voices").callCorrectMethod(queryVariables, null); + // assertEquals(expectedOutput, actualOutput); + // + // //expected output, locale=de, gender=male: [] + // expectedOutput = "[]"; + // queryVariables.put("locale", "de"); + // queryVariables.put("gender", "male"); + // actualOutput = requestHandler.methodsMapping.get("voices").callCorrectMethod(queryVariables, null); + // assertEquals(expectedOutput, actualOutput); + // } + // + // @Test + // public void testVoicesWithLocaleAndType() { + // InfoRequestHandler requestHandler = new InfoRequestHandler(); + // Map queryVariables = new HashMap (); + // + // //expected output, locale=en-us, type=hmm: [{"name":"cmu-slt-hsmm","locale":"en_US","gender":"female","type":"hmm"}] + // String expectedOutput = "[{\"name\":\"cmu-slt-hsmm\",\"locale\":\"en_US\",\"gender\":\"female\",\"type\":\"hmm\"}]"; + // queryVariables.put("locale", "en-us"); + // queryVariables.put("type", "hmm"); + // String actualOutput = requestHandler.methodsMapping.get("voices").callCorrectMethod(queryVariables, null); + // assertEquals(expectedOutput, actualOutput); + // + // //expected output, locale=de, type=hmm: [] + // expectedOutput = "[]"; + // queryVariables.put("locale", "de"); + // queryVariables.put("type", "hmm"); + // actualOutput = requestHandler.methodsMapping.get("voices").callCorrectMethod(queryVariables, null); + // assertEquals(expectedOutput, actualOutput); + // + // //expected output, locale=en-us, type=unit: [] + // expectedOutput = "[]"; + // queryVariables.put("locale", "en-us"); + // queryVariables.put("type", "unit"); + // actualOutput = requestHandler.methodsMapping.get("voices").callCorrectMethod(queryVariables, null); + // assertEquals(expectedOutput, actualOutput); + // + // //expected output, locale=de, type=hmm: [] + // expectedOutput = "[]"; + // queryVariables.put("locale", "de"); + // queryVariables.put("type", "unit"); + // actualOutput = requestHandler.methodsMapping.get("voices").callCorrectMethod(queryVariables, null); + // assertEquals(expectedOutput, actualOutput); + // } + // + // @Test + // public void testVoicesWithNameAndGender() { + // InfoRequestHandler requestHandler = new InfoRequestHandler(); + // Map queryVariables = new HashMap (); + // + // //expected output, name=cmu-slt-hsmm, gender=female: + // [{"name":"cmu-slt-hsmm","locale":"en_US","gender":"female","type":"hmm"}] + // String expectedOutput = "[{\"name\":\"cmu-slt-hsmm\",\"locale\":\"en_US\",\"gender\":\"female\",\"type\":\"hmm\"}]"; + // queryVariables.put("name", "cmu-slt-hsmm"); + // queryVariables.put("gender", "female"); + // String actualOutput = requestHandler.methodsMapping.get("voices").callCorrectMethod(queryVariables, null); + // assertEquals(expectedOutput, actualOutput); + // + // //expected output, name=cmu-slt-hsmm, gender=male: [] + // expectedOutput = "[]"; + // queryVariables.put("name", "cmu-slt-hsmm"); + // queryVariables.put("gender", "male"); + // actualOutput = requestHandler.methodsMapping.get("voices").callCorrectMethod(queryVariables, null); + // assertEquals(expectedOutput, actualOutput); + // + // //expected output, name=dummy-name, gender=female: [] + // expectedOutput = "[]"; + // queryVariables.put("name", "dummy-name"); + // queryVariables.put("gender", "female"); + // actualOutput = requestHandler.methodsMapping.get("voices").callCorrectMethod(queryVariables, null); + // assertEquals(expectedOutput, actualOutput); + // + // //expected output, name=dummy-name, gender=male: [] + // expectedOutput = "[]"; + // queryVariables.put("name", "dummy-name"); + // queryVariables.put("gender", "male"); + // actualOutput = requestHandler.methodsMapping.get("voices").callCorrectMethod(queryVariables, null); + // assertEquals(expectedOutput, actualOutput); + // } + // + // @Test + // public void testVoicesWithNameAndType() { + // InfoRequestHandler requestHandler = new InfoRequestHandler(); + // Map queryVariables = new HashMap (); + // + // //expected output, name=cmu-slt-hsmm, type=hmm: [{"name":"cmu-slt-hsmm","locale":"en_US","gender":"female","type":"hmm"}] + // String expectedOutput = "[{\"name\":\"cmu-slt-hsmm\",\"locale\":\"en_US\",\"gender\":\"female\",\"type\":\"hmm\"}]"; + // queryVariables.put("name", "cmu-slt-hsmm"); + // queryVariables.put("type", "hmm"); + // String actualOutput = requestHandler.methodsMapping.get("voices").callCorrectMethod(queryVariables, null); + // assertEquals(expectedOutput, actualOutput); + // + // //expected output, name=cmu-slt-hsmm, type=unit: [] + // expectedOutput = "[]"; + // queryVariables.put("name", "cmu-slt-hsmm"); + // queryVariables.put("type", "unit"); + // actualOutput = requestHandler.methodsMapping.get("voices").callCorrectMethod(queryVariables, null); + // assertEquals(expectedOutput, actualOutput); + // + // //expected output, name=dummy-name, type=hmm: [] + // expectedOutput = "[]"; + // queryVariables.put("name", "dummy-name"); + // queryVariables.put("type", "hmm"); + // actualOutput = requestHandler.methodsMapping.get("voices").callCorrectMethod(queryVariables, null); + // assertEquals(expectedOutput, actualOutput); + // + // //expected output, name=dummy-name, type=unit: [] + // expectedOutput = "[]"; + // queryVariables.put("name", "dummy-name"); + // queryVariables.put("type", "unit"); + // actualOutput = requestHandler.methodsMapping.get("voices").callCorrectMethod(queryVariables, null); + // assertEquals(expectedOutput, actualOutput); + // } + // + // @Test + // public void testVoicesWithLocaleNameAndGender() { + // InfoRequestHandler requestHandler = new InfoRequestHandler(); + // Map queryVariables = new HashMap (); + // + // //expected output, locale=en-us, name=cmu-slt-hsmm, gender=female: + // [{"name":"cmu-slt-hsmm","locale":"en_US","gender":"female","type":"hmm"}] + // String expectedOutput = "[{\"name\":\"cmu-slt-hsmm\",\"locale\":\"en_US\",\"gender\":\"female\",\"type\":\"hmm\"}]"; + // queryVariables.put("locale", "en-us"); + // queryVariables.put("name", "cmu-slt-hsmm"); + // queryVariables.put("gender", "female"); + // String actualOutput = requestHandler.methodsMapping.get("voices").callCorrectMethod(queryVariables, null); + // assertEquals(expectedOutput, actualOutput); + // + // //expected output, locale=en-us, name=cmu-slt-hsmm, gender=male: [] + // expectedOutput = "[]"; + // queryVariables.put("locale", "en-us"); + // queryVariables.put("name", "cmu-slt-hsmm"); + // queryVariables.put("gender", "male"); + // actualOutput = requestHandler.methodsMapping.get("voices").callCorrectMethod(queryVariables, null); + // assertEquals(expectedOutput, actualOutput); + // + // //expected output, locale=de, name=dummy-name, gender=female: [] + // expectedOutput = "[]"; + // queryVariables.put("locale", "de"); + // queryVariables.put("name", "dummy-name"); + // queryVariables.put("gender", "female"); + // actualOutput = requestHandler.methodsMapping.get("voices").callCorrectMethod(queryVariables, null); + // assertEquals(expectedOutput, actualOutput); + // + // //expected output, locale=de, name=dummy-name, gender=male: [] + // expectedOutput = "[]"; + // queryVariables.put("locale", "de"); + // queryVariables.put("name", "dummy-name"); + // queryVariables.put("gender", "male"); + // actualOutput = requestHandler.methodsMapping.get("voices").callCorrectMethod(queryVariables, null); + // assertEquals(expectedOutput, actualOutput); + // } + // + // @Test + // public void testVoicesWithLocaleNameAndType() { + // InfoRequestHandler requestHandler = new InfoRequestHandler(); + // Map queryVariables = new HashMap (); + // + // //expected output, locale=en-us, name=cmu-slt-hsmm, type=hmm: + // [{"name":"cmu-slt-hsmm","locale":"en_US","gender":"female","type":"hmm"}] + // String expectedOutput = "[{\"name\":\"cmu-slt-hsmm\",\"locale\":\"en_US\",\"gender\":\"female\",\"type\":\"hmm\"}]"; + // queryVariables.put("locale", "en-us"); + // queryVariables.put("name", "cmu-slt-hsmm"); + // queryVariables.put("type", "hmm"); + // String actualOutput = requestHandler.methodsMapping.get("voices").callCorrectMethod(queryVariables, null); + // assertEquals(expectedOutput, actualOutput); + // + // //expected output, locale=en-us, name=cmu-slt-hsmm, type=unit: [] + // expectedOutput = "[]"; + // queryVariables.put("locale", "en-us"); + // queryVariables.put("name", "cmu-slt-hsmm"); + // queryVariables.put("type", "unit"); + // actualOutput = requestHandler.methodsMapping.get("voices").callCorrectMethod(queryVariables, null); + // assertEquals(expectedOutput, actualOutput); + // + // //expected output, locale=de, name=dummy-name, type=hmm: [] + // expectedOutput = "[]"; + // queryVariables.put("locale", "de"); + // queryVariables.put("name", "dummy-name"); + // queryVariables.put("type", "hmm"); + // actualOutput = requestHandler.methodsMapping.get("voices").callCorrectMethod(queryVariables, null); + // assertEquals(expectedOutput, actualOutput); + // + // //expected output, locale=de, name=dummy-name, type=unit: [] + // expectedOutput = "[]"; + // queryVariables.put("locale", "de"); + // queryVariables.put("name", "dummy-name"); + // queryVariables.put("type", "unit"); + // actualOutput = requestHandler.methodsMapping.get("voices").callCorrectMethod(queryVariables, null); + // assertEquals(expectedOutput, actualOutput); + // } + // + // @Test + // public void testVoicesWithLocaleNameGenderAndType() { + // InfoRequestHandler requestHandler = new InfoRequestHandler(); + // Map queryVariables = new HashMap (); + // + // //expected output, locale=en-us, name=cmu-slt-hsmm, gender=female, type=hmm: + // [{"name":"cmu-slt-hsmm","locale":"en_US","gender":"female","type":"hmm"}] + // String expectedOutput = "[{\"name\":\"cmu-slt-hsmm\",\"locale\":\"en_US\",\"gender\":\"female\",\"type\":\"hmm\"}]"; + // queryVariables.put("locale", "en-us"); + // queryVariables.put("name", "cmu-slt-hsmm"); + // queryVariables.put("gender", "female"); + // queryVariables.put("type", "hmm"); + // String actualOutput = requestHandler.methodsMapping.get("voices").callCorrectMethod(queryVariables, null); + // assertEquals(expectedOutput, actualOutput); + // + // //expected output, locale=en-us, name=cmu-slt-hsmm, gender=female, type=unit: [] + // expectedOutput = "[]"; + // queryVariables.put("locale", "en-us"); + // queryVariables.put("name", "cmu-slt-hsmm"); + // queryVariables.put("gender", "female"); + // queryVariables.put("type", "unit"); + // actualOutput = requestHandler.methodsMapping.get("voices").callCorrectMethod(queryVariables, null); + // assertEquals(expectedOutput, actualOutput); + // + // //expected output, locale=de, name=dummy-name, gender=female, type=hmm: [] + // expectedOutput = "[]"; + // queryVariables.put("locale", "de"); + // queryVariables.put("name", "dummy-name"); + // queryVariables.put("gender", "female"); + // queryVariables.put("type", "hmm"); + // actualOutput = requestHandler.methodsMapping.get("voices").callCorrectMethod(queryVariables, null); + // assertEquals(expectedOutput, actualOutput); + // + // //expected output, locale=de, name=dummy-name, gender=male, type=hmm: [] + // expectedOutput = "[]"; + // queryVariables.put("locale", "de"); + // queryVariables.put("name", "dummy-name"); + // queryVariables.put("gender", "male"); + // queryVariables.put("type", "hmm"); + // actualOutput = requestHandler.methodsMapping.get("voices").callCorrectMethod(queryVariables, null); + // assertEquals(expectedOutput, actualOutput); + // + // //expected output, locale=de, name=dummy-name, gender=male, type=unit: [] + // expectedOutput = "[]"; + // queryVariables.put("locale", "de"); + // queryVariables.put("name", "dummy-name"); + // queryVariables.put("gender", "male"); + // queryVariables.put("type", "unit"); + // actualOutput = requestHandler.methodsMapping.get("voices").callCorrectMethod(queryVariables, null); + // assertEquals(expectedOutput, actualOutput); + // } + + @Test + public void testSortedVoiceJSON() { + InfoRequestHandler requestHandler = new InfoRequestHandler(); + + List voices; + voices = new ArrayList(Voice.getAvailableVoices()); + + if (voices.size() > 0) { + Voice dummyVoice = voices.get(0); + + Voice v1, v2, v3; + try { + v1 = new Voice(dummyVoice.getName(), dummyVoice.synthesizer()); + v1.setVoiceName("dummy3"); + v1.setLocale(new Locale("en-GB")); + voices.add(v1); + + v2 = new Voice(dummyVoice.getName(), dummyVoice.synthesizer()); + v2.setVoiceName("dummy1"); + voices.add(v2); + + v3 = new Voice(dummyVoice.getName(), dummyVoice.synthesizer()); + v3.setVoiceName("dummy2"); + voices.add(v3); + } catch (MaryConfigurationException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + + String expectedOutput = "[{\"name\":\"dummy3\",\"locale\":\"en-gb\",\"gender\":\"female\",\"type\":\"other\"},{\"name\":\"cmu-slt-hsmm\",\"locale\":\"en_US\",\"gender\":\"female\",\"type\":\"hmm\"},{\"name\":\"dummy1\",\"locale\":\"en_US\",\"gender\":\"female\",\"type\":\"other\"},{\"name\":\"dummy2\",\"locale\":\"en_US\",\"gender\":\"female\",\"type\":\"other\"}]"; + String actualOutput = requestHandler.methodsMapping.get("voices").callCorrectMethod(new HashMap(), + null); + assertEquals(expectedOutput, actualOutput); + } + } +} diff --git a/pom.xml b/pom.xml index 4700ee8c74..2aabd86510 100644 --- a/pom.xml +++ b/pom.xml @@ -233,7 +233,13 @@ diff4j 1.0 - + + + com.google.code.gson + gson + 2.3.1 + + com.google.guava guava From cd000f3104546953971a47cd9c89949f5ac3f30c Mon Sep 17 00:00:00 2001 From: Tristan Hamilton Date: Mon, 2 Feb 2015 15:03:28 +0100 Subject: [PATCH 02/31] created new empty marytts-unitselection maven module. updated master and builder pom --- marytts-builder/pom.xml | 5 +++++ marytts-unitselection/pom.xml | 18 ++++++++++++++++++ pom.xml | 1 + 3 files changed, 24 insertions(+) create mode 100644 marytts-unitselection/pom.xml diff --git a/marytts-builder/pom.xml b/marytts-builder/pom.xml index eb8d27ac3a..3c4b8727a4 100644 --- a/marytts-builder/pom.xml +++ b/marytts-builder/pom.xml @@ -37,6 +37,11 @@ marytts-signalproc ${project.version} + + ${project.groupId} + marytts-unitselection + ${project.version} + commons-lang diff --git a/marytts-unitselection/pom.xml b/marytts-unitselection/pom.xml new file mode 100644 index 0000000000..00b112dcbd --- /dev/null +++ b/marytts-unitselection/pom.xml @@ -0,0 +1,18 @@ + + 4.0.0 + + de.dfki.mary + marytts + 5.2-SNAPSHOT + + + marytts-unitselection + ${project.artifactId} + + + de.dfki.mary + marytts-runtime + 5.2-SNAPSHOT + + + \ No newline at end of file diff --git a/pom.xml b/pom.xml index 2aabd86510..d224bd9422 100644 --- a/pom.xml +++ b/pom.xml @@ -178,6 +178,7 @@ marytts-transcription voice-cmu-slt-hsmm marytts-assembly + marytts-unitselection From 69fd3efda1352f2fc7bcc922e3cd9df13b97a015 Mon Sep 17 00:00:00 2001 From: Tristan Hamilton Date: Mon, 2 Feb 2015 15:09:20 +0100 Subject: [PATCH 03/31] remove vocalizations package from runtime and remove specific vocalisationsFFR from unit selection package --- .../VocalizationFFRTargetCostFunction.java | 262 ------- .../FDPSOLASynthesisTechnology.java | 260 ------- .../vocalizations/HNMFeatureFileReader.java | 142 ---- .../vocalizations/HNMSynthesisTechnology.java | 298 -------- .../vocalizations/KMeansClusterer.java | 188 ------ .../vocalizations/MLSAFeatureFileReader.java | 273 -------- .../MLSASynthesisTechnology.java | 283 -------- .../vocalizations/SourceTargetPair.java | 65 -- .../vocalizations/VocalizationCandidate.java | 58 -- .../VocalizationFeatureFileReader.java | 89 --- .../VocalizationIntonationReader.java | 200 ------ .../vocalizations/VocalizationSelector.java | 639 ------------------ .../VocalizationSynthesisTechnology.java | 77 --- .../VocalizationSynthesizer.java | 305 --------- .../vocalizations/VocalizationUnit.java | 59 -- .../VocalizationUnitFileReader.java | 240 ------- 16 files changed, 3438 deletions(-) delete mode 100644 marytts-runtime/src/main/java/marytts/unitselection/select/VocalizationFFRTargetCostFunction.java delete mode 100644 marytts-runtime/src/main/java/marytts/vocalizations/FDPSOLASynthesisTechnology.java delete mode 100644 marytts-runtime/src/main/java/marytts/vocalizations/HNMFeatureFileReader.java delete mode 100644 marytts-runtime/src/main/java/marytts/vocalizations/HNMSynthesisTechnology.java delete mode 100644 marytts-runtime/src/main/java/marytts/vocalizations/KMeansClusterer.java delete mode 100644 marytts-runtime/src/main/java/marytts/vocalizations/MLSAFeatureFileReader.java delete mode 100644 marytts-runtime/src/main/java/marytts/vocalizations/MLSASynthesisTechnology.java delete mode 100644 marytts-runtime/src/main/java/marytts/vocalizations/SourceTargetPair.java delete mode 100644 marytts-runtime/src/main/java/marytts/vocalizations/VocalizationCandidate.java delete mode 100644 marytts-runtime/src/main/java/marytts/vocalizations/VocalizationFeatureFileReader.java delete mode 100644 marytts-runtime/src/main/java/marytts/vocalizations/VocalizationIntonationReader.java delete mode 100644 marytts-runtime/src/main/java/marytts/vocalizations/VocalizationSelector.java delete mode 100644 marytts-runtime/src/main/java/marytts/vocalizations/VocalizationSynthesisTechnology.java delete mode 100644 marytts-runtime/src/main/java/marytts/vocalizations/VocalizationSynthesizer.java delete mode 100644 marytts-runtime/src/main/java/marytts/vocalizations/VocalizationUnit.java delete mode 100644 marytts-runtime/src/main/java/marytts/vocalizations/VocalizationUnitFileReader.java diff --git a/marytts-runtime/src/main/java/marytts/unitselection/select/VocalizationFFRTargetCostFunction.java b/marytts-runtime/src/main/java/marytts/unitselection/select/VocalizationFFRTargetCostFunction.java deleted file mode 100644 index c6a0163eb2..0000000000 --- a/marytts-runtime/src/main/java/marytts/unitselection/select/VocalizationFFRTargetCostFunction.java +++ /dev/null @@ -1,262 +0,0 @@ -/** - * Copyright 2006 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.unitselection.select; - -import marytts.features.FeatureDefinition; -import marytts.features.FeatureVector; -import marytts.unitselection.data.Unit; -import marytts.unitselection.weightingfunctions.WeightFunc; -import marytts.unitselection.weightingfunctions.WeightFunctionManager; -import marytts.vocalizations.VocalizationFeatureFileReader; - -/** - * FFRTargetCostFunction for vocalization selection - * - * @author sathish pammi - * - */ -public class VocalizationFFRTargetCostFunction extends FFRTargetCostFunction { - - private int MEANING_RATING_RANGE = 5; // the range of meaning rating scale - - public VocalizationFFRTargetCostFunction(VocalizationFeatureFileReader ffr) { - this(ffr, ffr.getFeatureDefinition()); - } - - public VocalizationFFRTargetCostFunction(VocalizationFeatureFileReader ffr, FeatureDefinition fDef) { - load(ffr, fDef); - } - - /** - * load feature file reader and feature definition for a cost function - * - * @param ffr - * feature file reader - * @param fDef - * feature definition - */ - private void load(VocalizationFeatureFileReader ffr, FeatureDefinition fDef) { - this.featureVectors = ffr.featureVectorMapping(fDef); - this.featureDefinition = fDef; - - weightFunction = new WeightFunc[featureDefinition.getNumberOfContinuousFeatures()]; - WeightFunctionManager wfm = new WeightFunctionManager(); - int nDiscreteFeatures = featureDefinition.getNumberOfByteFeatures() + featureDefinition.getNumberOfShortFeatures(); - for (int i = 0; i < weightFunction.length; i++) { - String weightFunctionName = featureDefinition.getWeightFunctionName(nDiscreteFeatures + i); - if ("".equals(weightFunctionName)) - weightFunction[i] = wfm.getWeightFunction("linear"); - else - weightFunction[i] = wfm.getWeightFunction(weightFunctionName); - } - - rememberWhichWeightsAreNonZero(); - } - - /** - * Compute the goodness-of-fit of a given unit for a given target - * - * @param target - * target unit - * @param unit - * candidate unit - * @param weights - * FeatureDefinition - * @param weightFunctions - * array of WeightFunctions - * @return a non-negative number; smaller values mean better fit, i.e. smaller cost. - * @throws IllegalArgumentException - * if featureName not available in featureDefinition - */ - protected double cost(Target target, Unit unit, FeatureDefinition weights, WeightFunc[] weightFunctions) { - nCostComputations++; // for debug - FeatureVector targetFeatures = target.getFeatureVector(); - assert targetFeatures != null : "Target " + target + " does not have pre-computed feature vector"; - FeatureVector unitFeatures = featureVectors[unit.index]; - int nBytes = targetFeatures.byteValuedDiscreteFeatures.length; - int nShorts = targetFeatures.shortValuedDiscreteFeatures.length; - int nFloats = targetFeatures.continuousFeatures.length; - assert nBytes == unitFeatures.byteValuedDiscreteFeatures.length; - assert nShorts == unitFeatures.shortValuedDiscreteFeatures.length; - assert nFloats == unitFeatures.continuousFeatures.length; - - float[] weightVector = weights.getFeatureWeights(); - // Now the actual computation - double cost = 0; - // byte-valued features: - if (nBytes > 0) { - for (int i = 0; i < nBytes; i++) { - if (weightsNonZero[i]) { - float weight = weightVector[i]; - if (featureDefinition.hasSimilarityMatrix(i)) { - byte targetFeatValueIndex = targetFeatures.byteValuedDiscreteFeatures[i]; - byte unitFeatValueIndex = unitFeatures.byteValuedDiscreteFeatures[i]; - float similarity = featureDefinition.getSimilarity(i, unitFeatValueIndex, targetFeatValueIndex); - cost += similarity * weight; - if (debugShowCostGraph) - cumulWeightedCosts[i] += similarity * weight; - } else if (targetFeatures.byteValuedDiscreteFeatures[i] != unitFeatures.byteValuedDiscreteFeatures[i]) { - cost += weight; - if (debugShowCostGraph) - cumulWeightedCosts[i] += weight; - } - } - } - } - // short-valued features: - if (nShorts > 0) { - for (int i = nBytes, n = nBytes + nShorts; i < n; i++) { - if (weightsNonZero[i]) { - float weight = weightVector[i]; - // if (targetFeatures.getShortFeature(i) != unitFeatures.getShortFeature(i)) { - if (targetFeatures.shortValuedDiscreteFeatures[i - nBytes] != unitFeatures.shortValuedDiscreteFeatures[i - - nBytes]) { - cost += weight; - if (debugShowCostGraph) - cumulWeightedCosts[i] += weight; - } - } - } - } - // continuous features: - if (nFloats > 0) { - int nDiscrete = nBytes + nShorts; - for (int i = nDiscrete, n = nDiscrete + nFloats; i < n; i++) { - if (weightsNonZero[i]) { - float weight = weightVector[i]; - // float a = targetFeatures.getContinuousFeature(i); - float a = targetFeatures.continuousFeatures[i - nDiscrete]; - // float b = unitFeatures.getContinuousFeature(i); - float b = unitFeatures.continuousFeatures[i - nDiscrete]; - // if (!Float.isNaN(a) && !Float.isNaN(b)) { - // Implementation of isNaN() is: (v != v). - if (!(a != a)) { - - double myCost; - if (!(b != b)) { - myCost = weightFunctions[i - nDiscrete].cost(a, b); - } else { - myCost = this.MEANING_RATING_RANGE; - } - - cost += weight * myCost; - if (debugShowCostGraph) { - cumulWeightedCosts[i] += weight * myCost; - } - } // and if it is NaN, simply compute no cost - } - } - } - return cost; - } - - /** - * Compute the goodness-of-fit between given unit and given target for a given feature - * - * @param target - * target unit - * @param unit - * candidate unit - * @param featureName - * feature name - * @param weights - * FeatureDefinition - * @param weightFunctions - * array of WeightFunctions - * @return a non-negative number; smaller values mean better fit, i.e. smaller cost. - * @throws IllegalArgumentException - * if featureName not available in featureDefinition - */ - protected double featureCost(Target target, Unit unit, String featureName, FeatureDefinition weights, - WeightFunc[] weightFunctions) { - if (!this.featureDefinition.hasFeature(featureName)) { - throw new IllegalArgumentException("this feature does not exists in feature definition"); - } - - FeatureVector targetFeatures = target.getFeatureVector(); - assert targetFeatures != null : "Target " + target + " does not have pre-computed feature vector"; - FeatureVector unitFeatures = featureVectors[unit.index]; - int nBytes = targetFeatures.byteValuedDiscreteFeatures.length; - int nShorts = targetFeatures.shortValuedDiscreteFeatures.length; - int nFloats = targetFeatures.continuousFeatures.length; - assert nBytes == unitFeatures.byteValuedDiscreteFeatures.length; - assert nShorts == unitFeatures.shortValuedDiscreteFeatures.length; - assert nFloats == unitFeatures.continuousFeatures.length; - - int featureIndex = this.featureDefinition.getFeatureIndex(featureName); - float[] weightVector = weights.getFeatureWeights(); - double cost = 0; - - if (featureIndex < nBytes) { - if (weightsNonZero[featureIndex]) { - float weight = weightVector[featureIndex]; - if (featureDefinition.hasSimilarityMatrix(featureIndex)) { - byte targetFeatValueIndex = targetFeatures.byteValuedDiscreteFeatures[featureIndex]; - byte unitFeatValueIndex = unitFeatures.byteValuedDiscreteFeatures[featureIndex]; - float similarity = featureDefinition.getSimilarity(featureIndex, unitFeatValueIndex, targetFeatValueIndex); - cost = similarity * weight; - if (debugShowCostGraph) - cumulWeightedCosts[featureIndex] += similarity * weight; - } else if (targetFeatures.byteValuedDiscreteFeatures[featureIndex] != unitFeatures.byteValuedDiscreteFeatures[featureIndex]) { - cost = weight; - if (debugShowCostGraph) - cumulWeightedCosts[featureIndex] += weight; - } - } - } else if (featureIndex < nShorts + nBytes) { - if (weightsNonZero[featureIndex]) { - float weight = weightVector[featureIndex]; - // if (targetFeatures.getShortFeature(i) != unitFeatures.getShortFeature(i)) { - if (targetFeatures.shortValuedDiscreteFeatures[featureIndex - nBytes] != unitFeatures.shortValuedDiscreteFeatures[featureIndex - - nBytes]) { - cost = weight; - if (debugShowCostGraph) - cumulWeightedCosts[featureIndex] += weight; - } - } - } else { - int nDiscrete = nBytes + nShorts; - if (weightsNonZero[featureIndex]) { - float weight = weightVector[featureIndex]; - // float a = targetFeatures.getContinuousFeature(i); - float a = targetFeatures.continuousFeatures[featureIndex - nDiscrete]; - // float b = unitFeatures.getContinuousFeature(i); - float b = unitFeatures.continuousFeatures[featureIndex - nDiscrete]; - // if (!Float.isNaN(a) && !Float.isNaN(b)) { - // Implementation of isNaN() is: (v != v). - if (!(a != a)) { - - double myCost; - if (!(b != b)) { - myCost = weightFunctions[featureIndex - nDiscrete].cost(a, b); - } else { - myCost = this.MEANING_RATING_RANGE; - } - - cost = weight * myCost; - if (debugShowCostGraph) { - cumulWeightedCosts[featureIndex] += weight * myCost; - } - } // and if it is NaN, simply compute no cost - } - } - return cost; - } -} diff --git a/marytts-runtime/src/main/java/marytts/vocalizations/FDPSOLASynthesisTechnology.java b/marytts-runtime/src/main/java/marytts/vocalizations/FDPSOLASynthesisTechnology.java deleted file mode 100644 index 2d91fba6cd..0000000000 --- a/marytts-runtime/src/main/java/marytts/vocalizations/FDPSOLASynthesisTechnology.java +++ /dev/null @@ -1,260 +0,0 @@ -/** - * Copyright 2000-2006 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.vocalizations; - -import java.io.IOException; -import java.util.Arrays; -import java.util.LinkedList; - -import javax.sound.sampled.AudioFileFormat; -import javax.sound.sampled.AudioFormat; -import javax.sound.sampled.AudioInputStream; - -import marytts.exceptions.MaryConfigurationException; -import marytts.exceptions.SynthesisException; -import marytts.signalproc.process.FDPSOLAProcessor; -import marytts.unitselection.data.TimelineReader; -import marytts.util.data.BufferedDoubleDataSource; -import marytts.util.data.Datagram; -import marytts.util.data.DatagramDoubleDataSource; -import marytts.util.data.DoubleDataSource; -import marytts.util.data.audio.DDSAudioInputStream; -import marytts.util.math.MathUtils; -import marytts.util.math.Polynomial; - -/** - * FDPSOLA Synthesis technology to synthesize vocalizations - * - * @author Sathish Pammi - */ - -public class FDPSOLASynthesisTechnology extends VocalizationSynthesisTechnology { - - protected VocalizationIntonationReader vIntonationReader; - protected TimelineReader audioTimeline; - protected VocalizationUnitFileReader unitFileReader; - protected boolean f0ContourImposeSupport; - - public FDPSOLASynthesisTechnology(String waveTimeLineFile, String unitFile, String intonationFeatureFile, - boolean imposeF0Support) throws MaryConfigurationException { - - try { - this.audioTimeline = new TimelineReader(waveTimeLineFile); - this.unitFileReader = new VocalizationUnitFileReader(unitFile); - this.f0ContourImposeSupport = imposeF0Support; - - if (f0ContourImposeSupport) { - this.vIntonationReader = new VocalizationIntonationReader(intonationFeatureFile); - } else { - this.vIntonationReader = null; - } - } catch (IOException e) { - throw new MaryConfigurationException("Can not read data from files " + e); - } - } - - public FDPSOLASynthesisTechnology(TimelineReader audioTimeline, VocalizationUnitFileReader unitFileReader, - HNMFeatureFileReader vHNMFeaturesReader, VocalizationIntonationReader vIntonationReader, boolean imposeF0Support) { - - this.audioTimeline = audioTimeline; - this.unitFileReader = unitFileReader; - this.vIntonationReader = vIntonationReader; - this.f0ContourImposeSupport = imposeF0Support; - - } - - /** - * Synthesize given vocalization (i.e. unit-selection) - * - * @param unitIndex - * unit index - * @param aft - * audio file format - * @return AudioInputStream of synthesized vocalization - * @throws SynthesisException - * if failed to synthesize vocalization - */ - @Override - public AudioInputStream synthesize(int backchannelNumber, AudioFileFormat aft) throws SynthesisException { - - int numberOfBackChannels = unitFileReader.getNumberOfUnits(); - if (backchannelNumber >= numberOfBackChannels) { - throw new IllegalArgumentException("This voice has " + numberOfBackChannels - + " backchannels only. so it doesn't support unit number " + backchannelNumber); - } - - VocalizationUnit bUnit = unitFileReader.getUnit(backchannelNumber); - long start = bUnit.startTime; - int duration = bUnit.duration; - Datagram[] frames = null; - try { - frames = audioTimeline.getDatagrams(start, duration); - } catch (IOException e) { - throw new SynthesisException("Can not read data from timeline file " + e); - } - // Generate audio from frames - LinkedList datagrams = new LinkedList(); - datagrams.addAll(Arrays.asList(frames)); - DoubleDataSource audioSource = new DatagramDoubleDataSource(datagrams); - // audioSource.getAllData(); - return (new DDSAudioInputStream(new BufferedDoubleDataSource(audioSource), aft.getFormat())); - } - - /** - * Re-synthesize given vocalization using FDPSOLA technology - * - * @param unitIndex - * unit index - * @param aft - * audio file format - * @return AudioInputStream of synthesized vocalization - * @throws SynthesisException - * if failed to synthesize vocalization - */ - public AudioInputStream reSynthesize(int backchannelNumber, AudioFileFormat aft) throws SynthesisException { - double[] pScalesArray = { 1.0f }; - double[] tScalesArray = { 1.0f }; - return synthesizeUsingF0Modification(backchannelNumber, pScalesArray, tScalesArray, aft); - } - - /** - * Impose target intonation contour on given vocalization using HNM technology - * - * @param sourceIndex - * unit index of vocalization - * @param targetIndex - * unit index of target intonation - * @param aft - * audio file format - * @return AudioInputStream of synthesized vocalization - * @throws SynthesisException - * if failed to synthesize vocalization - */ - @Override - public AudioInputStream synthesizeUsingImposedF0(int sourceIndex, int targetIndex, AudioFileFormat aft) - throws SynthesisException { - - if (!f0ContourImposeSupport) { - throw new SynthesisException("Mary configuration of this voice doesn't support intonation contour imposition"); - } - - int numberOfUnits = unitFileReader.getNumberOfUnits(); - if (sourceIndex >= numberOfUnits || targetIndex >= numberOfUnits) { - throw new IllegalArgumentException("sourceIndex(" + sourceIndex + ") and targetIndex(" + targetIndex - + ") are should be less than number of available units (" + numberOfUnits + ")"); - } - - if (sourceIndex == targetIndex) { - return reSynthesize(sourceIndex, aft); - } - - double[] sourceF0 = this.vIntonationReader.getContour(sourceIndex); - double[] targetF0coeffs = this.vIntonationReader.getIntonationCoeffs(targetIndex); - double[] sourceF0coeffs = this.vIntonationReader.getIntonationCoeffs(sourceIndex); - - if (targetF0coeffs == null || sourceF0coeffs == null) { - return reSynthesize(sourceIndex, aft); - } - - if (targetF0coeffs.length == 0 || sourceF0coeffs.length == 0) { - return reSynthesize(sourceIndex, aft); - } - - double[] targetF0 = Polynomial.generatePolynomialValues(targetF0coeffs, sourceF0.length, 0, 1); - sourceF0 = Polynomial.generatePolynomialValues(sourceF0coeffs, sourceF0.length, 0, 1); - - assert targetF0.length == sourceF0.length; - double[] tScalesArray = new double[sourceF0.length]; - double[] pScalesArray = new double[sourceF0.length]; - - for (int i = 0; i < targetF0.length; i++) { - pScalesArray[i] = (float) (targetF0[i] / sourceF0[i]); - tScalesArray[i] = (float) (1.0); - } - - return synthesizeUsingF0Modification(sourceIndex, pScalesArray, tScalesArray, aft); - } - - /** - * modify intonation contour using HNM technology - * - * @param backchannelNumber - * unit index of vocalization - * @param pScalesArray - * pitch scales array - * @param tScalesArray - * time scales array - * @param aft - * audio file format - * @return AudioInputStream of synthesized vocalization - * @throws SynthesisException - * if failed to synthesize vocalization - */ - private AudioInputStream synthesizeUsingF0Modification(int backchannelNumber, double[] pScalesArray, double[] tScalesArray, - AudioFileFormat aft) throws SynthesisException { - - if (backchannelNumber > unitFileReader.getNumberOfUnits()) { - throw new IllegalArgumentException("requesting unit should not be more than number of units"); - } - - if (!f0ContourImposeSupport) { - throw new SynthesisException("Mary configuration of this voice doesn't support intonation contour imposition"); - } - - VocalizationUnit bUnit = unitFileReader.getUnit(backchannelNumber); - long start = bUnit.startTime; - int duration = bUnit.duration; - Datagram[] frames = null; - try { - frames = audioTimeline.getDatagrams(start, duration); - } catch (IOException e) { - throw new SynthesisException("cannot get audio frames from timeline file " + e); - } - assert frames != null : "Cannot generate audio from null frames"; - - pScalesArray = MathUtils.arrayResize(pScalesArray, frames.length); - tScalesArray = MathUtils.arrayResize(tScalesArray, frames.length); - - assert tScalesArray.length == pScalesArray.length; - assert frames.length == tScalesArray.length; - - AudioFormat af; - if (aft == null) { // default audio format - float sampleRate = 16000.0F; // 8000,11025,16000,22050,44100 - int sampleSizeInBits = 16; // 8,16 - int channels = 1; // 1,2 - boolean signed = true; // true,false - boolean bigEndian = false; // true,false - af = new AudioFormat(sampleRate, sampleSizeInBits, channels, signed, bigEndian); - } else { - af = aft.getFormat(); - } - - double[] audio_double = (new FDPSOLAProcessor()).processDatagram(frames, null, af, null, pScalesArray, tScalesArray, - false); - /* Normalise the signal before return, this will normalise between 1 and -1 */ - double MaxSample = MathUtils.getAbsMax(audio_double); - for (int i = 0; i < audio_double.length; i++) { - audio_double[i] = 0.3 * (audio_double[i] / MaxSample); - } - return (new DDSAudioInputStream(new BufferedDoubleDataSource(audio_double), af)); - } - -} diff --git a/marytts-runtime/src/main/java/marytts/vocalizations/HNMFeatureFileReader.java b/marytts-runtime/src/main/java/marytts/vocalizations/HNMFeatureFileReader.java deleted file mode 100644 index 20b6a3a9d4..0000000000 --- a/marytts-runtime/src/main/java/marytts/vocalizations/HNMFeatureFileReader.java +++ /dev/null @@ -1,142 +0,0 @@ -/** - * Portions Copyright 2006 DFKI GmbH. - * Portions Copyright 2001 Sun Microsystems, Inc. - * Portions Copyright 1999-2001 Language Technologies Institute, - * Carnegie Mellon University. - * All Rights Reserved. Use is subject to license terms. - * - * Permission is hereby granted, free of charge, to use and distribute - * this software and its documentation without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of this work, and to - * permit persons to whom this work is furnished to do so, subject to - * the following conditions: - * - * 1. The code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * 2. Any modifications must be clearly marked as such. - * 3. Original authors' names are not deleted. - * 4. The authors' names are not used to endorse or promote products - * derived from this software without specific prior written - * permission. - * - * DFKI GMBH AND THE CONTRIBUTORS TO THIS WORK DISCLAIM ALL WARRANTIES WITH - * REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL DFKI GMBH NOR THE - * CONTRIBUTORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL - * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR - * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS - * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF - * THIS SOFTWARE. - */ -package marytts.vocalizations; - -import java.io.BufferedInputStream; -import java.io.DataInputStream; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.IOException; - -import marytts.exceptions.MaryConfigurationException; -import marytts.signalproc.sinusoidal.hntm.analysis.HntmAnalyzerParams; -import marytts.signalproc.sinusoidal.hntm.analysis.HntmSpeechSignal; -import marytts.util.data.MaryHeader; - -/** - * Reads a single file which contains HNM analysis features of vocalizations - * - * @author sathish pammi - * - */ -public class HNMFeatureFileReader { - - private MaryHeader hdr = null; - private HntmSpeechSignal[] hnmSignals; - private int numberOfUnits = 0; - - /** - * Create a feature file reader from the given HNM feature file - * - * @param fileName - * the unit file to read - * @throws IOException - * if a problem occurs while reading - * @throws MaryConfigurationException - * if runtime configuration fails - */ - public HNMFeatureFileReader(String fileName) throws IOException, MaryConfigurationException { - load(fileName); - } - - /** - * Load the given feature file - * - * @param fileName - * the feature file to read - * @throws IOException - * if a problem occurs while reading - * @throws MaryConfigurationException - * if runtime configuration fails - */ - private void load(String fileName) throws IOException, MaryConfigurationException { - // Open the file - DataInputStream dis = null; - - try { - dis = new DataInputStream(new BufferedInputStream(new FileInputStream(fileName))); - } catch (FileNotFoundException e) { - throw new MaryConfigurationException("File [" + fileName + "] was not found."); - } - - // Load the Mary header - hdr = new MaryHeader(dis); - if (hdr.getType() != MaryHeader.LISTENERFEATS) { - throw new MaryConfigurationException("File [" + fileName + "] is not a valid Mary Units file."); - } - - numberOfUnits = dis.readInt(); // Read the number of units - if (numberOfUnits < 0) { - throw new MaryConfigurationException("File [" + fileName + "] has a negative number of units. Aborting."); - } - - hnmSignals = new HntmSpeechSignal[numberOfUnits]; - for (int i = 0; i < numberOfUnits; i++) { - hnmSignals[i] = new HntmSpeechSignal(0, 0, 0); - hnmSignals[i].read(dis, HntmAnalyzerParams.WAVEFORM); - } - - System.out.println(); - } - - /** - * Get the number of units in the file. - * - * @return The number of units. - */ - public int getNumberOfUnits() { - return (numberOfUnits); - } - - /** - * get HntmSpeechSignal for a unit index - * - * @param unitnumber - * unit index number - * @return HntmSpeechSignal hnm analysis feature - * @throws IllegalArgumentException - * if given index number is not less than available units - */ - public HntmSpeechSignal getHntmSpeechSignal(int unitnumber) { - if (unitnumber >= this.numberOfUnits) { - throw new IllegalArgumentException("the given unit index number(" + unitnumber - + ") must be less than number of available units(" + this.numberOfUnits + ")"); - } - return this.hnmSignals[unitnumber]; - } - - public static void main(String[] args) throws Exception { - String fileName = "/home/sathish/Work/phd/voices/mlsa-poppy-listener/vocalizations/files/vocalization_hnm_analysis.mry"; - HNMFeatureFileReader bcUfr = new HNMFeatureFileReader(fileName); - // bcUfr.load(fileName); - } -} diff --git a/marytts-runtime/src/main/java/marytts/vocalizations/HNMSynthesisTechnology.java b/marytts-runtime/src/main/java/marytts/vocalizations/HNMSynthesisTechnology.java deleted file mode 100644 index c22dff689e..0000000000 --- a/marytts-runtime/src/main/java/marytts/vocalizations/HNMSynthesisTechnology.java +++ /dev/null @@ -1,298 +0,0 @@ -/** - * Copyright 2000-2006 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.vocalizations; - -import java.io.IOException; -import java.util.Arrays; -import java.util.LinkedList; - -import javax.sound.sampled.AudioFileFormat; -import javax.sound.sampled.AudioFormat; -import javax.sound.sampled.AudioInputStream; - -import marytts.exceptions.MaryConfigurationException; -import marytts.exceptions.SynthesisException; -import marytts.signalproc.adaptation.prosody.BasicProsodyModifierParams; -import marytts.signalproc.analysis.RegularizedCepstrumEstimator; -import marytts.signalproc.sinusoidal.hntm.analysis.HntmAnalyzerParams; -import marytts.signalproc.sinusoidal.hntm.analysis.HntmSpeechSignal; -import marytts.signalproc.sinusoidal.hntm.synthesis.HntmSynthesizedSignal; -import marytts.signalproc.sinusoidal.hntm.synthesis.HntmSynthesizer; -import marytts.signalproc.sinusoidal.hntm.synthesis.HntmSynthesizerParams; -import marytts.unitselection.data.TimelineReader; -import marytts.util.data.BufferedDoubleDataSource; -import marytts.util.data.Datagram; -import marytts.util.data.DatagramDoubleDataSource; -import marytts.util.data.DoubleDataSource; -import marytts.util.data.audio.DDSAudioInputStream; -import marytts.util.math.MathUtils; -import marytts.util.math.Polynomial; - -/** - * HNM Synthesis technology to synthesize vocalizations - * - * @author Sathish Pammi - */ - -public class HNMSynthesisTechnology extends VocalizationSynthesisTechnology { - - protected HNMFeatureFileReader vHNMFeaturesReader; - protected VocalizationIntonationReader vIntonationReader; - protected HntmAnalyzerParams analysisParams; - protected HntmSynthesizerParams synthesisParams; - protected TimelineReader audioTimeline; - protected VocalizationUnitFileReader unitFileReader; - protected boolean f0ContourImposeSupport; - - public HNMSynthesisTechnology(String waveTimeLineFile, String unitFile, String hnmFeatureFile, String intonationFeatureFile, - boolean imposeF0Support) throws MaryConfigurationException { - - try { - this.audioTimeline = new TimelineReader(waveTimeLineFile); - this.unitFileReader = new VocalizationUnitFileReader(unitFile); - this.f0ContourImposeSupport = imposeF0Support; - this.vHNMFeaturesReader = new HNMFeatureFileReader(hnmFeatureFile); - - if (f0ContourImposeSupport) { - this.vIntonationReader = new VocalizationIntonationReader(intonationFeatureFile); - } else { - this.vIntonationReader = null; - } - } catch (IOException e) { - throw new MaryConfigurationException("Can not read data from files " + e); - } - - initializeParameters(); - } - - public HNMSynthesisTechnology(TimelineReader audioTimeline, VocalizationUnitFileReader unitFileReader, - HNMFeatureFileReader vHNMFeaturesReader, VocalizationIntonationReader vIntonationReader, boolean imposeF0Support) { - - this.audioTimeline = audioTimeline; - this.unitFileReader = unitFileReader; - this.vHNMFeaturesReader = vHNMFeaturesReader; - this.vIntonationReader = vIntonationReader; - this.f0ContourImposeSupport = imposeF0Support; - - initializeParameters(); - } - - /** - * intialize hnm parameters - */ - private void initializeParameters() { - // Analysis parameters - analysisParams = new HntmAnalyzerParams(); - analysisParams.harmonicModel = HntmAnalyzerParams.HARMONICS_PLUS_NOISE; - analysisParams.noiseModel = HntmAnalyzerParams.WAVEFORM; - analysisParams.useHarmonicAmplitudesDirectly = true; - analysisParams.harmonicSynthesisMethodBeforeNoiseAnalysis = HntmSynthesizerParams.LINEAR_PHASE_INTERPOLATION; - analysisParams.regularizedCepstrumWarpingMethod = RegularizedCepstrumEstimator.REGULARIZED_CEPSTRUM_WITH_POST_MEL_WARPING; - - // Synthesis parameters - synthesisParams = new HntmSynthesizerParams(); - synthesisParams.harmonicPartSynthesisMethod = HntmSynthesizerParams.LINEAR_PHASE_INTERPOLATION; - // synthesisParams.harmonicPartSynthesisMethod = HntmSynthesizerParams.QUADRATIC_PHASE_INTERPOLATION; - synthesisParams.overlappingHarmonicPartSynthesis = false; - synthesisParams.harmonicSynthesisOverlapInSeconds = 0.010f; - /* to output just one file */ - synthesisParams.writeHarmonicPartToSeparateFile = false; - synthesisParams.writeNoisePartToSeparateFile = false; - synthesisParams.writeTransientPartToSeparateFile = false; - synthesisParams.writeOriginalMinusHarmonicPartToSeparateFile = false; - } - - /** - * Synthesize given vocalization (i.e. unit-selection) - * - * @param unitIndex - * unit index - * @param aft - * audio file format - * @return AudioInputStream of synthesized vocalization - * @throws SynthesisException - * if failed to synthesize vocalization - */ - @Override - public AudioInputStream synthesize(int backchannelNumber, AudioFileFormat aft) throws SynthesisException { - - int numberOfBackChannels = unitFileReader.getNumberOfUnits(); - if (backchannelNumber >= numberOfBackChannels) { - throw new IllegalArgumentException("This voice has " + numberOfBackChannels - + " backchannels only. so it doesn't support unit number " + backchannelNumber); - } - - VocalizationUnit bUnit = unitFileReader.getUnit(backchannelNumber); - long start = bUnit.startTime; - int duration = bUnit.duration; - Datagram[] frames = null; - try { - frames = audioTimeline.getDatagrams(start, duration); - } catch (IOException e) { - throw new SynthesisException("Can not read data from timeline file " + e); - } - // Generate audio from frames - LinkedList datagrams = new LinkedList(); - datagrams.addAll(Arrays.asList(frames)); - DoubleDataSource audioSource = new DatagramDoubleDataSource(datagrams); - // audioSource.getAllData(); - return (new DDSAudioInputStream(new BufferedDoubleDataSource(audioSource), aft.getFormat())); - } - - /** - * Re-synthesize given vocalization using HNM technology - * - * @param unitIndex - * unit index - * @param aft - * audio file format - * @return AudioInputStream of synthesized vocalization - * @throws SynthesisException - * if failed to synthesize vocalization - */ - @Override - public AudioInputStream reSynthesize(int backchannelNumber, AudioFileFormat aft) throws SynthesisException { - float[] pScalesArray = { 1.0f }; - float[] tScalesArray = { 1.0f }; - float[] tScalesTimes = { 1.0f }; - float[] pScalesTimes = { 1.0f }; - return synthesizeUsingF0Modification(backchannelNumber, pScalesArray, pScalesTimes, tScalesArray, tScalesTimes, aft); - } - - /** - * Impose target intonation contour on given vocalization using HNM technology - * - * @param sourceIndex - * unit index of vocalization - * @param targetIndex - * unit index of target intonation - * @param aft - * audio file format - * @return AudioInputStream of synthesized vocalization - * @throws SynthesisException - * if failed to synthesize vocalization - */ - @Override - public AudioInputStream synthesizeUsingImposedF0(int sourceIndex, int targetIndex, AudioFileFormat aft) - throws SynthesisException { - - if (!f0ContourImposeSupport) { - throw new SynthesisException("Mary configuration of this voice doesn't support intonation contour imposition"); - } - - int numberOfUnits = vHNMFeaturesReader.getNumberOfUnits(); - if (sourceIndex >= numberOfUnits || targetIndex >= numberOfUnits) { - throw new IllegalArgumentException("sourceIndex(" + sourceIndex + ") and targetIndex(" + targetIndex - + ") are should be less than number of available units (" + numberOfUnits + ")"); - } - - double[] sourceF0 = this.vIntonationReader.getContour(sourceIndex); - double[] targetF0coeffs = this.vIntonationReader.getIntonationCoeffs(targetIndex); - double[] sourceF0coeffs = this.vIntonationReader.getIntonationCoeffs(sourceIndex); - - if (targetF0coeffs == null || sourceF0coeffs == null) { - return reSynthesize(sourceIndex, aft); - } - - if (targetF0coeffs.length == 0 || sourceF0coeffs.length == 0) { - return reSynthesize(sourceIndex, aft); - } - - double[] targetF0 = Polynomial.generatePolynomialValues(targetF0coeffs, sourceF0.length, 0, 1); - sourceF0 = Polynomial.generatePolynomialValues(sourceF0coeffs, sourceF0.length, 0, 1); - - assert targetF0.length == sourceF0.length; - float[] tScalesArray = { 1.0f }; - float[] tScalesTimes = { 1.0f }; - float[] pScalesArray = new float[targetF0.length]; - float[] pScalesTimes = new float[targetF0.length]; - double skipSizeInSeconds = this.vIntonationReader.getSkipSizeInSeconds(); - double windowSizeInSeconds = this.vIntonationReader.getWindowSizeInSeconds(); - for (int i = 0; i < targetF0.length; i++) { - pScalesArray[i] = (float) (targetF0[i] / sourceF0[i]); - pScalesTimes[i] = (float) (i * skipSizeInSeconds + 0.5 * windowSizeInSeconds); - } - - return synthesizeUsingF0Modification(sourceIndex, pScalesArray, pScalesTimes, tScalesArray, tScalesTimes, aft); - } - - /** - * modify intonation contour using HNM technology - * - * @param backchannelNumber - * unit index of vocalization - * @param pScalesArray - * pitch scales array - * @param pScalesTimes - * pitch scale times - * @param tScalesArray - * time scales array - * @param tScalesTimes - * time scale times - * @param aft - * audio file format - * @return AudioInputStream of synthesized vocalization - * @throws SynthesisException - * if failed to synthesize vocalization - */ - private AudioInputStream synthesizeUsingF0Modification(int backchannelNumber, float[] pScalesArray, float[] pScalesTimes, - float[] tScalesArray, float[] tScalesTimes, AudioFileFormat aft) throws SynthesisException { - - if (backchannelNumber > vHNMFeaturesReader.getNumberOfUnits()) { - throw new IllegalArgumentException("requesting unit should not be more than number of units"); - } - - if (!f0ContourImposeSupport) { - throw new SynthesisException("Mary configuration of this voice doesn't support intonation contour imposition"); - } - - BasicProsodyModifierParams pmodParams = new BasicProsodyModifierParams(tScalesArray, tScalesTimes, pScalesArray, - pScalesTimes); // Prosody from modification factors above - - HntmSpeechSignal hnmSignal = vHNMFeaturesReader.getHntmSpeechSignal(backchannelNumber); - HntmSynthesizer hs = new HntmSynthesizer(); - HntmSynthesizedSignal xhat = hs.synthesize(hnmSignal, null, null, pmodParams, null, analysisParams, synthesisParams); - - AudioFormat af; - if (aft == null) { // default audio format - float sampleRate = 16000.0F; // 8000,11025,16000,22050,44100 - int sampleSizeInBits = 16; // 8,16 - int channels = 1; // 1,2 - boolean signed = true; // true,false - boolean bigEndian = false; // true,false - af = new AudioFormat(sampleRate, sampleSizeInBits, channels, signed, bigEndian); - } else { - af = aft.getFormat(); - } - - double[] audio_double = xhat.output; - /* Normalise the signal before return, this will normalise between 1 and -1 */ - double MaxSample = MathUtils.getAbsMax(audio_double); - for (int i = 0; i < audio_double.length; i++) { - audio_double[i] = 0.3 * (audio_double[i] / MaxSample); - } - - // DDSAudioInputStream oais = new DDSAudioInputStream(new BufferedDoubleDataSource(audio_double), aft.getFormat()); - DDSAudioInputStream oais = new DDSAudioInputStream(new BufferedDoubleDataSource(audio_double), af); - - return oais; - } - -} diff --git a/marytts-runtime/src/main/java/marytts/vocalizations/KMeansClusterer.java b/marytts-runtime/src/main/java/marytts/vocalizations/KMeansClusterer.java deleted file mode 100644 index 80785cce12..0000000000 --- a/marytts-runtime/src/main/java/marytts/vocalizations/KMeansClusterer.java +++ /dev/null @@ -1,188 +0,0 @@ -/** - * Copyright 2010 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.vocalizations; - -import java.awt.Color; -import java.io.BufferedReader; -import java.io.File; -import java.io.FileReader; -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; - -import javax.sound.sampled.AudioFormat; -import javax.sound.sampled.AudioInputStream; -import javax.sound.sampled.AudioSystem; -import javax.sound.sampled.UnsupportedAudioFileException; -import javax.swing.JFrame; - -import marytts.machinelearning.KMeansClusteringTrainerParams; -import marytts.machinelearning.PolynomialCluster; -import marytts.machinelearning.PolynomialKMeansClusteringTrainer; -import marytts.signalproc.display.FunctionGraph; -import marytts.util.data.BufferedDoubleDataSource; -import marytts.util.data.audio.AudioDoubleDataSource; -import marytts.util.data.audio.AudioPlayer; -import marytts.util.data.audio.DDSAudioInputStream; -import marytts.util.math.Polynomial; - -public class KMeansClusterer { - - private ArrayList baseNames; - Polynomial[] f0Polynomials; - private int polynomialOrder = 3; - private int numberOfSamples = 0; - private int numberOfClusters; - private HashMap mapPolynomialBaseNames; - - public KMeansClusterer() { - baseNames = new ArrayList(); - mapPolynomialBaseNames = new HashMap(); - } - - public void loadF0Polynomials(String fileName) throws IOException { - - BufferedReader bfr = new BufferedReader(new FileReader(new File(fileName))); - String line; - ArrayList lines = new ArrayList(); - while ((line = bfr.readLine()) != null) { - lines.add(line.trim()); - } - double[][] f0PolynomialCoeffs; - String[] words = lines.get(0).trim().split("\\s+"); - polynomialOrder = words.length - 1; - numberOfSamples = lines.size(); - f0PolynomialCoeffs = new double[numberOfSamples][polynomialOrder]; - f0Polynomials = new Polynomial[numberOfSamples]; - - for (int i = 0; i < lines.size(); i++) { - line = lines.get(i); - words = line.trim().split("\\s+"); - baseNames.add(words[0].trim()); - - for (int j = 0; j < polynomialOrder; j++) { - f0PolynomialCoeffs[i][j] = (new Double(words[j + 1].trim())).doubleValue(); - } - - // System.out.println(f0PolynomialCoeffs[i][0]+" "+f0PolynomialCoeffs[i][1]+" "+f0PolynomialCoeffs[i][2]+" "+f0PolynomialCoeffs[i][3]); - // System.out.println(words[0]+" "+words[1]+" "+words[2]+" "+words[3]+" "+words[4]); - } - - // testing - for (int i = 0; i < numberOfSamples; i++) { - f0Polynomials[i] = new Polynomial(f0PolynomialCoeffs[i]); - mapPolynomialBaseNames.put(f0Polynomials[i], baseNames.get(i)); - System.out.print(baseNames.get(i) + " "); - for (int j = 0; j < polynomialOrder; j++) { - System.out.print(f0PolynomialCoeffs[i][j] + " "); - } - System.out.println(); - - double coeff[] = f0Polynomials[i].coeffs; - System.out.print(baseNames.get(i) + " "); - for (int j = 0; j < coeff.length; j++) { - System.out.print(coeff[j] + " "); - } - System.out.println(); - } - - } - - public void trainer(int numClusters) throws UnsupportedAudioFileException, IOException { - this.numberOfClusters = numClusters; - KMeansClusteringTrainerParams params = new KMeansClusteringTrainerParams(); - params.numClusters = numClusters; - params.maxIterations = 10000; - // Train: - PolynomialCluster[] clusters = PolynomialKMeansClusteringTrainer.train(f0Polynomials, params); - - // Visualise: - FunctionGraph clusterGraph = new FunctionGraph(0, 1, new double[1]); - clusterGraph.setYMinMax(-550, 500); - clusterGraph.setPrimaryDataSeriesStyle(Color.BLUE, FunctionGraph.DRAW_DOTS, FunctionGraph.DOT_FULLCIRCLE); - JFrame jf = clusterGraph.showInJFrame("", false, true); - for (int i = 0; i < clusters.length; i++) { - double[] meanValues = clusters[i].getMeanPolynomial().generatePolynomialValues(100, 0, 1); - clusterGraph.updateData(0, 1. / meanValues.length, meanValues); - - Polynomial[] members = clusters[i].getClusterMembers(); - System.out.print("Cluster " + i + " : "); - for (int m = 0; m < members.length; m++) { - double[] pred = members[m].generatePolynomialValues(meanValues.length, 0, 1); - clusterGraph.addDataSeries(pred, Color.GRAY, FunctionGraph.DRAW_LINE, -1); - String baseName = mapPolynomialBaseNames.get(members[m]); - System.out.print(baseName + " "); - jf.repaint(); - - String waveFile = "/home/sathish/Work/phd/voices/f0desc-listener/vocalizations/wav/" + File.separator + baseName - + ".wav"; - AudioInputStream inputAudio = AudioSystem.getAudioInputStream(new File(waveFile)); - - // Enforce PCM_SIGNED encoding - if (!inputAudio.getFormat().getEncoding().equals(AudioFormat.Encoding.PCM_SIGNED)) { - inputAudio = AudioSystem.getAudioInputStream(AudioFormat.Encoding.PCM_SIGNED, inputAudio); - } - - int audioSampleRate = (int) inputAudio.getFormat().getSampleRate(); - AudioDoubleDataSource signal = new AudioDoubleDataSource(inputAudio); - double[] sentenceAudio = signal.getAllData(); - AudioPlayer ap = new AudioPlayer(new DDSAudioInputStream(new BufferedDoubleDataSource(sentenceAudio), - new AudioFormat(AudioFormat.Encoding.PCM_SIGNED, audioSampleRate, // samples per second - 16, // bits per sample - 1, // mono - 2, // nr. of bytes per frame - audioSampleRate, // nr. of frames per second - true))); // big-endian;)) - ap.start(); - - try { - ap.join(); - Thread.sleep(10); - } catch (InterruptedException ie) { - } - - } - System.out.println(); - jf.setTitle("Cluster " + (i + 1) + " of " + clusters.length + ": " + members.length + " members"); - jf.repaint(); - - try { - Thread.sleep(5000); - } catch (InterruptedException ie) { - } - } - } - - /** - * @param args - * @throws IOException - * @throws UnsupportedAudioFileException - */ - public static void main(String[] args) throws IOException, UnsupportedAudioFileException { - KMeansClusterer kmc = new KMeansClusterer(); - // String fileName = "/home/sathish/phd/voices/en-GB-listener/vocal-polynomials/all.listener.polynomials.txt"; - // String fileName = "/home/sathish/phd/voices/en-GB-listener/vocal-polynomials/SpiVocalizationF0PolyFeatureFile.txt"; - String fileName = "/home/sathish/phd/voices/en-GB-listener/yeahPrudenceVocalizationF0PolyFeatureFile.txt"; - kmc.loadF0Polynomials(fileName); - kmc.trainer(5); - System.exit(0); - } - -} diff --git a/marytts-runtime/src/main/java/marytts/vocalizations/MLSAFeatureFileReader.java b/marytts-runtime/src/main/java/marytts/vocalizations/MLSAFeatureFileReader.java deleted file mode 100644 index 7998516349..0000000000 --- a/marytts-runtime/src/main/java/marytts/vocalizations/MLSAFeatureFileReader.java +++ /dev/null @@ -1,273 +0,0 @@ -/** - * Portions Copyright 2006 DFKI GmbH. - * Portions Copyright 2001 Sun Microsystems, Inc. - * Portions Copyright 1999-2001 Language Technologies Institute, - * Carnegie Mellon University. - * All Rights Reserved. Use is subject to license terms. - * - * Permission is hereby granted, free of charge, to use and distribute - * this software and its documentation without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of this work, and to - * permit persons to whom this work is furnished to do so, subject to - * the following conditions: - * - * 1. The code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * 2. Any modifications must be clearly marked as such. - * 3. Original authors' names are not deleted. - * 4. The authors' names are not used to endorse or promote products - * derived from this software without specific prior written - * permission. - * - * DFKI GMBH AND THE CONTRIBUTORS TO THIS WORK DISCLAIM ALL WARRANTIES WITH - * REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL DFKI GMBH NOR THE - * CONTRIBUTORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL - * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR - * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS - * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF - * THIS SOFTWARE. - */ -package marytts.vocalizations; - -import java.io.BufferedInputStream; -import java.io.DataInputStream; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.IOException; - -import marytts.exceptions.MaryConfigurationException; -import marytts.util.data.MaryHeader; - -/** - * Reads a single file which contains all MLSA features (logfo, mgc and strengths) of vocalizations - * - * @author sathish pammi - * - */ -public class MLSAFeatureFileReader { - - private MaryHeader hdr = null; - - private int numberOfUnits; - private int STRVECTORSIZE; - private int MGCVECTORSIZE; - private int LF0VECTORSIZE; - - private int[] numberOfFrames; - private boolean[][] voiced; - private double[][] logf0; - private double[][][] strengths; - private double[][][] mgc; - - /** - * Create a feature file reader from the given MLSA feature file - * - * @param fileName - * the unit file to read - * @throws IOException - * if a problem occurs while reading - * @throws MaryConfigurationException - * if runtime configuration fails - */ - public MLSAFeatureFileReader(String fileName) throws IOException, MaryConfigurationException { - load(fileName); - } - - /** - * Load the given feature file - * - * @param fileName - * the feature file to read - * @throws IOException - * if a problem occurs while reading - * @throws MaryConfigurationException - * if runtime configuration fails - */ - private void load(String fileName) throws IOException, MaryConfigurationException { - // Open the file - DataInputStream dis = null; - - try { - dis = new DataInputStream(new BufferedInputStream(new FileInputStream(fileName))); - } catch (FileNotFoundException e) { - throw new MaryConfigurationException("File [" + fileName + "] was not found."); - } - - // Load the Mary header - hdr = new MaryHeader(dis); - if (hdr.getType() != MaryHeader.LISTENERFEATS) { - throw new MaryConfigurationException("File [" + fileName + "] is not a valid Mary Units file."); - } - - numberOfUnits = dis.readInt(); // Read the number of units - if (numberOfUnits < 0) { - throw new MaryConfigurationException("File [" + fileName + "] has a negative number of units. Aborting."); - } - - LF0VECTORSIZE = dis.readInt(); // Read LF0 vector size - MGCVECTORSIZE = dis.readInt(); // Read MGC vector size - STRVECTORSIZE = dis.readInt(); // Read STR vector size - - if (LF0VECTORSIZE != 1 || MGCVECTORSIZE <= 0 || STRVECTORSIZE <= 0) { - throw new MaryConfigurationException("File [" + fileName - + "] has no proper feature vector size information... Aborting."); - } - - logf0 = new double[numberOfUnits][]; - voiced = new boolean[numberOfUnits][]; - mgc = new double[numberOfUnits][][]; - strengths = new double[numberOfUnits][][]; - numberOfFrames = new int[numberOfUnits]; - - for (int i = 0; i < numberOfUnits; i++) { - - numberOfFrames[i] = dis.readInt(); - - // read LF0 data - int checkLF0Size = dis.readInt(); - assert checkLF0Size == (numberOfFrames[i] * LF0VECTORSIZE) : fileName + " feature file do not has proper format"; - logf0[i] = new double[numberOfFrames[i]]; - voiced[i] = new boolean[numberOfFrames[i]]; - for (int j = 0; j < numberOfFrames[i]; j++) { - logf0[i][j] = dis.readFloat(); - if (logf0[i][j] < 0) { - voiced[i][j] = false; - } else { - voiced[i][j] = true; - } - } - - // read MGC data - int checkMGCSize = dis.readInt(); - assert checkMGCSize == (numberOfFrames[i] * this.MGCVECTORSIZE) : fileName + " feature file do not has proper format"; - mgc[i] = new double[numberOfFrames[i]][MGCVECTORSIZE]; - for (int j = 0; j < numberOfFrames[i]; j++) { - for (int k = 0; k < MGCVECTORSIZE; k++) { - mgc[i][j][k] = dis.readFloat(); - } - } - - // read STR data - int checkSTRSize = dis.readInt(); - assert checkSTRSize == (numberOfFrames[i] * this.STRVECTORSIZE) : fileName + " feature file do not has proper format"; - strengths[i] = new double[numberOfFrames[i]][STRVECTORSIZE]; - for (int j = 0; j < numberOfFrames[i]; j++) { - for (int k = 0; k < STRVECTORSIZE; k++) { - strengths[i][j][k] = dis.readFloat(); - } - } - } - } - - /** - * Get the number of units in the file. - * - * @return The number of units. - */ - public int getNumberOfUnits() { - return (numberOfUnits); - } - - /** - * get boolean array of voiced frame information: true, if voiced; false if unvoiced; - * - * @param unitnumber - * unit index number - * @return boolean[] boolean array of voiced frames - * @throws IllegalArgumentException - * if given index number is not less than available units - */ - public boolean[] getVoicedFrames(int unitnumber) { - if (unitnumber >= this.numberOfUnits) { - throw new IllegalArgumentException("the given unit index number(" + unitnumber - + ") must be less than number of available units(" + this.numberOfUnits + ")"); - } - return this.voiced[unitnumber]; - } - - /** - * get array of logf0 features - * - * @param unitnumber - * unit index number - * @return double[] array of logf0 values - * @throws IllegalArgumentException - * if given index number is not less than available units - */ - public double[] getUnitLF0(int unitnumber) { - if (unitnumber >= this.numberOfUnits) { - throw new IllegalArgumentException("the given unit index number(" + unitnumber - + ") must be less than number of available units(" + this.numberOfUnits + ")"); - } - return this.logf0[unitnumber]; - } - - /** - * get double array of MGC features - * - * @param unitnumber - * unit index number - * @return double[][] array of mgc vectors - * @throws IllegalArgumentException - * if given index number is not less than available units - */ - public double[][] getUnitMGCs(int unitnumber) { - if (unitnumber >= this.numberOfUnits) { - throw new IllegalArgumentException("the given unit index number(" + unitnumber - + ") must be less than number of available units(" + this.numberOfUnits + ")"); - } - return this.mgc[unitnumber]; - } - - /** - * get double array of strength features - * - * @param unitnumber - * unit index number - * @return double[][] array of strength vectors - * @throws IllegalArgumentException - * if given index number is not less than available units - */ - public double[][] getUnitStrengths(int unitnumber) { - if (unitnumber >= this.numberOfUnits) { - throw new IllegalArgumentException("the given unit index number(" + unitnumber - + ") must be less than number of available units(" + this.numberOfUnits + ")"); - } - return this.strengths[unitnumber]; - } - - /** - * get vector size of MGC features - * - * @return int mgc vector size - */ - public int getMGCVectorSize() { - return this.MGCVECTORSIZE; - } - - /** - * get vector size of LF0 features - * - * @return int lf0 vector size - */ - public int getLF0VectorSize() { - return this.LF0VECTORSIZE; - } - - /** - * get vector size of strength features - * - * @return int strengths vector size - */ - public int getSTRVectorSize() { - return this.STRVECTORSIZE; - } - - public static void main(String[] args) throws Exception { - String fileName = "/home/sathish/Work/phd/voices/mlsa-poppy-listener/vocalizations/files/vocalization_mlsa_features.mry"; - MLSAFeatureFileReader bcUfr = new MLSAFeatureFileReader(fileName); - // bcUfr.load(fileName); - } -} diff --git a/marytts-runtime/src/main/java/marytts/vocalizations/MLSASynthesisTechnology.java b/marytts-runtime/src/main/java/marytts/vocalizations/MLSASynthesisTechnology.java deleted file mode 100644 index 9064dbe7a4..0000000000 --- a/marytts-runtime/src/main/java/marytts/vocalizations/MLSASynthesisTechnology.java +++ /dev/null @@ -1,283 +0,0 @@ -/** - * Copyright 2000-2006 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.vocalizations; - -import java.io.FileInputStream; -import java.io.IOException; - -import javax.sound.sampled.AudioFileFormat; -import javax.sound.sampled.AudioFormat; -import javax.sound.sampled.AudioInputStream; - -import marytts.exceptions.MaryConfigurationException; -import marytts.exceptions.SynthesisException; -import marytts.htsengine.HMMData; -import marytts.htsengine.HTSPStream; -import marytts.htsengine.HTSVocoder; -import marytts.util.data.BufferedDoubleDataSource; -import marytts.util.data.audio.DDSAudioInputStream; -import marytts.util.math.MathUtils; -import marytts.util.math.Polynomial; - -/** - * MLSA Synthesis technology to synthesize vocalizations - * - * @author Sathish Pammi - */ - -public class MLSASynthesisTechnology extends VocalizationSynthesisTechnology { - - protected MLSAFeatureFileReader vMLSAFeaturesReader; - protected VocalizationIntonationReader vIntonationReader; - protected HMMData htsData; - protected HTSVocoder par2speech; - protected boolean imposePolynomialContour = true; - - public MLSASynthesisTechnology(String mlsaFeatureFile, String intonationFeatureFile, String mixedExcitationFile, - boolean imposePolynomialContour) throws MaryConfigurationException { - - try { - vMLSAFeaturesReader = new MLSAFeatureFileReader(mlsaFeatureFile); - if (intonationFeatureFile != null) { - this.vIntonationReader = new VocalizationIntonationReader(intonationFeatureFile); - } else { - this.vIntonationReader = null; - } - } catch (IOException ioe) { - throw new MaryConfigurationException("Problem with loading mlsa feature file", ioe); - } - - if (vMLSAFeaturesReader.getNumberOfUnits() <= 0) { - throw new MaryConfigurationException("mlsa feature file doesn't contain any data"); - } - - this.imposePolynomialContour = imposePolynomialContour; - - try { - htsData = new HMMData(); - htsData.setUseMixExc(true); - htsData.setUseFourierMag(false); /* use Fourier magnitudes for pulse generation */ - FileInputStream mixedFiltersStream = new FileInputStream(mixedExcitationFile); - htsData.setNumFilters(5); - htsData.setOrderFilters(48); - htsData.readMixedExcitationFilters(mixedFiltersStream); - htsData.setPdfStrStream(null); - // [min][max] - htsData.setF0Std(1.0); // variable for f0 control, multiply f0 [1.0][0.0--5.0] - htsData.setF0Mean(0.0); // variable for f0 control, add f0 [0.0][0.0--100.0] - } catch (Exception e) { - throw new MaryConfigurationException("htsData initialization failed.. ", e); - } - - par2speech = new HTSVocoder(); - - } - - /** - * Synthesize given vocalization using MLSA vocoder - * - * @param unitIndex - * unit index - * @param aft - * audio file format - * @return AudioInputStream of synthesized vocalization - * @throws SynthesisException - * if failed to synthesize vocalization - */ - public AudioInputStream synthesize(int backchannelNumber, AudioFileFormat aft) throws SynthesisException { - - if (backchannelNumber > vMLSAFeaturesReader.getNumberOfUnits()) { - throw new IllegalArgumentException("requesting unit should not be more than number of units"); - } - - if (backchannelNumber < 0) { - throw new IllegalArgumentException("requesting unit index should not be less than zero"); - } - - double[] lf0 = vMLSAFeaturesReader.getUnitLF0(backchannelNumber); - boolean[] voiced = vMLSAFeaturesReader.getVoicedFrames(backchannelNumber); - double[][] mgc = vMLSAFeaturesReader.getUnitMGCs(backchannelNumber); - double[][] strengths = vMLSAFeaturesReader.getUnitStrengths(backchannelNumber); - - return synthesizeUsingMLSAVocoder(mgc, strengths, lf0, voiced, aft); - } - - /** - * Re-synthesize given vocalization using MLSA (it is same as synthesize()) - * - * @param unitIndex - * unit index - * @param aft - * audio file format - * @return AudioInputStream of synthesized vocalization - * @throws SynthesisException - * if failed to synthesize vocalization - */ - @Override - public AudioInputStream reSynthesize(int backchannelNumber, AudioFileFormat aft) throws SynthesisException { - return synthesize(backchannelNumber, aft); - } - - /** - * Impose target intonation contour on given vocalization using MLSA technology - * - * @param sourceIndex - * unit index of vocalization - * @param targetIndex - * unit index of target intonation - * @param aft - * audio file format - * @return AudioInputStream of synthesized vocalization - * @throws SynthesisException - * if failed to synthesize vocalization - */ - @Override - public AudioInputStream synthesizeUsingImposedF0(int sourceIndex, int targetIndex, AudioFileFormat aft) - throws SynthesisException { - - if (sourceIndex > vMLSAFeaturesReader.getNumberOfUnits() || targetIndex > vMLSAFeaturesReader.getNumberOfUnits()) { - throw new IllegalArgumentException("requesting unit should not be more than number of units"); - } - - if (sourceIndex < 0 || targetIndex < 0) { - throw new IllegalArgumentException("requesting unit index should not be less than zero"); - } - - boolean[] voiced = vMLSAFeaturesReader.getVoicedFrames(sourceIndex); - double[][] mgc = vMLSAFeaturesReader.getUnitMGCs(sourceIndex); - double[][] strengths = vMLSAFeaturesReader.getUnitStrengths(sourceIndex); - - double[] lf0 = null; - - if (!this.imposePolynomialContour) { - lf0 = MathUtils.arrayResize(vMLSAFeaturesReader.getUnitLF0(targetIndex), voiced.length); - } else { - double[] targetF0coeffs = this.vIntonationReader.getIntonationCoeffs(targetIndex); - double[] sourceF0coeffs = this.vIntonationReader.getIntonationCoeffs(sourceIndex); - if (targetF0coeffs == null || sourceF0coeffs == null) { - return reSynthesize(sourceIndex, aft); - } - - if (targetF0coeffs.length == 0 || sourceF0coeffs.length == 0) { - return reSynthesize(sourceIndex, aft); - } - double[] f0Contour = Polynomial.generatePolynomialValues(targetF0coeffs, voiced.length, 0, 1); - lf0 = new double[f0Contour.length]; - for (int i = 0; i < f0Contour.length; i++) { - lf0[i] = Math.log(f0Contour[i]); - } - } - - return synthesizeUsingMLSAVocoder(mgc, strengths, lf0, voiced, aft); - } - - /** - * Synthesize using MLSA vocoder - * - * @param mgc - * mgc features - * @param strengths - * strengths - * @param lf0 - * logf0 features - * @param voiced - * voiced frames - * @param aft - * audio file format - * @return AudioInputStream of synthesized vocalization - * @throws SynthesisException - * if failed to synthesize vocalization - * @throws SynthesisException - * if log f0 values for voiced frames are not in the natural pitch range ( 30 to 1000 in Hertzs) - */ - private AudioInputStream synthesizeUsingMLSAVocoder(double[][] mgc, double[][] strengths, double[] lf0, boolean[] voiced, - AudioFileFormat aft) throws SynthesisException { - - assert lf0.length == mgc.length; - assert mgc.length == strengths.length; - - for (int i = 0; i < lf0.length; i++) { - if (lf0[i] > 0 && (Math.log(30) > lf0[i] || Math.log(1000) < lf0[i])) { - throw new SynthesisException("given log f0 values should be in the natural pitch range "); - } - } - - int mcepVsize = vMLSAFeaturesReader.getMGCVectorSize(); - int lf0Vsize = vMLSAFeaturesReader.getLF0VectorSize(); - int strVsize = vMLSAFeaturesReader.getSTRVectorSize(); - - HTSPStream lf0Pst = null; - HTSPStream mcepPst = null; - HTSPStream strPst = null; - - try { - lf0Pst = new HTSPStream(lf0Vsize * 3, lf0.length, HMMData.FeatureType.LF0, 0); // multiplied by 3 required for - // real-time synthesis - mcepPst = new HTSPStream(mcepVsize * 3, mgc.length, HMMData.FeatureType.MGC, 0); - strPst = new HTSPStream(strVsize * 3, strengths.length, HMMData.FeatureType.STR, 0); - } catch (Exception e) { - throw new SynthesisException("HTSPStream initialiaztion failed.. " + e); - } - - int lf0VoicedFrame = 0; - for (int i = 0; i < lf0.length; i++) { - if (voiced[i]) { - lf0Pst.setPar(lf0VoicedFrame, 0, lf0[i]); - lf0VoicedFrame++; - } - - for (int j = 0; j < mcepPst.getOrder(); j++) { - mcepPst.setPar(i, j, mgc[i][j]); - } - - for (int j = 0; j < strPst.getOrder(); j++) { - strPst.setPar(i, j, strengths[i][j]); - } - } - - AudioFormat af; - if (aft == null) { // default audio format - float sampleRate = 16000.0F; // 8000,11025,16000,22050,44100 - int sampleSizeInBits = 16; // 8,16 - int channels = 1; // 1,2 - boolean signed = true; // true,false - boolean bigEndian = false; // true,false - af = new AudioFormat(sampleRate, sampleSizeInBits, channels, signed, bigEndian); - } else { - af = aft.getFormat(); - } - - double[] audio_double = null; - try { - audio_double = par2speech.htsMLSAVocoder(lf0Pst, mcepPst, strPst, null, voiced, htsData, null); - } catch (Exception e) { - throw new SynthesisException("MLSA vocoding failed .. " + e); - } - - /* Normalise the signal before return, this will normalise between 1 and -1 */ - double MaxSample = MathUtils.getAbsMax(audio_double); - for (int i = 0; i < audio_double.length; i++) { - audio_double[i] = 0.3 * (audio_double[i] / MaxSample); - } - - return new DDSAudioInputStream(new BufferedDoubleDataSource(audio_double), af); - } - -} diff --git a/marytts-runtime/src/main/java/marytts/vocalizations/SourceTargetPair.java b/marytts-runtime/src/main/java/marytts/vocalizations/SourceTargetPair.java deleted file mode 100644 index a5e79d1d03..0000000000 --- a/marytts-runtime/src/main/java/marytts/vocalizations/SourceTargetPair.java +++ /dev/null @@ -1,65 +0,0 @@ -/** - * Copyright 2010 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.vocalizations; - -/** - * Class represents Source unit, target unit and contour distance between these units. Array of these pairs can sort with contour - * distance - * - * @author sathish - * - */ -public class SourceTargetPair implements Comparable { - - private int targetUnitIndex; - private int sourceUnitIndex; - private double distance; - - public SourceTargetPair(int sourceUnitIndex, int targetUnitIndex, double distance) { - this.sourceUnitIndex = sourceUnitIndex; - this.targetUnitIndex = targetUnitIndex; - this.distance = distance; - } - - public int compareTo(SourceTargetPair other) { - if (distance == other.distance) - return 0; - if (distance < other.distance) - return -1; - return 1; - } - - public boolean equals(Object dc) { - if (!(dc instanceof SourceTargetPair)) - return false; - SourceTargetPair other = (SourceTargetPair) dc; - if (distance == other.distance) - return true; - return false; - } - - public int getTargetUnitIndex() { - return this.targetUnitIndex; - } - - public int getSourceUnitIndex() { - return this.sourceUnitIndex; - } -} diff --git a/marytts-runtime/src/main/java/marytts/vocalizations/VocalizationCandidate.java b/marytts-runtime/src/main/java/marytts/vocalizations/VocalizationCandidate.java deleted file mode 100644 index 523fc2cb3f..0000000000 --- a/marytts-runtime/src/main/java/marytts/vocalizations/VocalizationCandidate.java +++ /dev/null @@ -1,58 +0,0 @@ -/** - * Copyright 2010 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.vocalizations; - -/** - * Class represents a vocalization candidate - * - * @author sathish - */ -public class VocalizationCandidate implements Comparable { - - int unitIndex; - double cost; - - public VocalizationCandidate(int unitIndex, double cost) { - this.unitIndex = unitIndex; - this.cost = cost; - } - - public int compareTo(VocalizationCandidate other) { - if (cost == other.cost) - return 0; - if (cost < other.cost) - return -1; - return 1; - } - - public boolean equals(Object dc) { - if (!(dc instanceof VocalizationCandidate)) - return false; - VocalizationCandidate other = (VocalizationCandidate) dc; - if (cost == other.cost) - return true; - return false; - } - - public String toString() { - return unitIndex + " " + cost; - } - -} diff --git a/marytts-runtime/src/main/java/marytts/vocalizations/VocalizationFeatureFileReader.java b/marytts-runtime/src/main/java/marytts/vocalizations/VocalizationFeatureFileReader.java deleted file mode 100644 index a66044cf0a..0000000000 --- a/marytts-runtime/src/main/java/marytts/vocalizations/VocalizationFeatureFileReader.java +++ /dev/null @@ -1,89 +0,0 @@ -/** - * Copyright 2010 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.vocalizations; - -import java.io.BufferedInputStream; -import java.io.DataInputStream; -import java.io.FileInputStream; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.channels.FileChannel; - -import marytts.exceptions.MaryConfigurationException; -import marytts.features.FeatureDefinition; -import marytts.features.FeatureVector; -import marytts.util.data.MaryHeader; - -public class VocalizationFeatureFileReader extends marytts.unitselection.data.FeatureFileReader { - - public VocalizationFeatureFileReader(String fileName) throws IOException, MaryConfigurationException { - load(fileName); - } - - @Override - protected void loadFromStream(String fileName) throws IOException, MaryConfigurationException { - /* Open the file */ - DataInputStream dis = null; - dis = new DataInputStream(new BufferedInputStream(new FileInputStream(fileName))); - - /* Load the Mary header */ - hdr = new MaryHeader(dis); - if (hdr.getType() != MaryHeader.LISTENERFEATS) { - throw new MaryConfigurationException("File [" + fileName + "] is not a valid Mary listener feature file."); - } - featureDefinition = new FeatureDefinition(dis); - int numberOfUnits = dis.readInt(); - featureVectors = new FeatureVector[numberOfUnits]; - for (int i = 0; i < numberOfUnits; i++) { - featureVectors[i] = featureDefinition.readFeatureVector(i, dis); - } - } - - @Override - protected void loadFromByteBuffer(String fileName) throws IOException, MaryConfigurationException { - /* Open the file */ - /* Open the file */ - FileInputStream fis = new FileInputStream(fileName); - FileChannel fc = fis.getChannel(); - ByteBuffer bb = fc.map(FileChannel.MapMode.READ_ONLY, 0, fc.size()); - fis.close(); - - /* Load the Mary header */ - hdr = new MaryHeader(bb); - if (hdr.getType() != MaryHeader.LISTENERFEATS) { - throw new MaryConfigurationException("File [" + fileName + "] is not a valid Mary listener feature file."); - } - featureDefinition = new FeatureDefinition(bb); - int numberOfUnits = bb.getInt(); - featureVectors = new FeatureVector[numberOfUnits]; - for (int i = 0; i < numberOfUnits; i++) { - featureVectors[i] = featureDefinition.readFeatureVector(i, bb); - } - } - - /** - * @param args - */ - public static void main(String[] args) { - // TODO Auto-generated method stub - - } - -} diff --git a/marytts-runtime/src/main/java/marytts/vocalizations/VocalizationIntonationReader.java b/marytts-runtime/src/main/java/marytts/vocalizations/VocalizationIntonationReader.java deleted file mode 100644 index 74a1c5690a..0000000000 --- a/marytts-runtime/src/main/java/marytts/vocalizations/VocalizationIntonationReader.java +++ /dev/null @@ -1,200 +0,0 @@ -/** - * Portions Copyright 2006 DFKI GmbH. - * Portions Copyright 2001 Sun Microsystems, Inc. - * Portions Copyright 1999-2001 Language Technologies Institute, - * Carnegie Mellon University. - * All Rights Reserved. Use is subject to license terms. - * - * Permission is hereby granted, free of charge, to use and distribute - * this software and its documentation without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of this work, and to - * permit persons to whom this work is furnished to do so, subject to - * the following conditions: - * - * 1. The code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * 2. Any modifications must be clearly marked as such. - * 3. Original authors' names are not deleted. - * 4. The authors' names are not used to endorse or promote products - * derived from this software without specific prior written - * permission. - * - * DFKI GMBH AND THE CONTRIBUTORS TO THIS WORK DISCLAIM ALL WARRANTIES WITH - * REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL DFKI GMBH NOR THE - * CONTRIBUTORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL - * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR - * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS - * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF - * THIS SOFTWARE. - */ -package marytts.vocalizations; - -import java.io.BufferedInputStream; -import java.io.DataInputStream; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.util.Arrays; - -import marytts.exceptions.MaryConfigurationException; -import marytts.util.data.MaryHeader; - -/** - * Vocalization contours and their corresponding polynomial coeffs reader from a intonation timeline reader file - * - * @author sathish pammi - * - */ -public class VocalizationIntonationReader { - - private MaryHeader hdr = null; - private int numberOfUnits = 0; - private double windowSize = 0.0; - private double skipSize = 0.0; - private double[][] contours; - private double[][] coeffs; - private boolean[][] voiced; - - /****************/ - /* CONSTRUCTORS */ - /****************/ - - /** - * Create a unit file reader from the given unit file - * - * @param fileName - * the unit file to read - * @throws IOException - * if a problem occurs while reading - */ - public VocalizationIntonationReader(String fileName) throws IOException, MaryConfigurationException { - load(fileName); - } - - /** - * Load the given unit file - * - * @param fileName - * the unit file to read - * @throws IOException - * if a problem occurs while reading - */ - public void load(String fileName) throws IOException, MaryConfigurationException { - /* Open the file */ - DataInputStream dis = null; - - try { - dis = new DataInputStream(new BufferedInputStream(new FileInputStream(fileName))); - } catch (FileNotFoundException e) { - throw new RuntimeException("File [" + fileName + "] was not found."); - } - - try { - /* Load the Mary header */ - hdr = new MaryHeader(dis); - if (hdr.getType() != MaryHeader.LISTENERFEATS) { - throw new RuntimeException("File [" + fileName + "] is not a valid Mary Units file."); - } - - windowSize = dis.readFloat(); - - if (this.windowSize <= 0) { - throw new RuntimeException("File [" + fileName + "] has a negative number for window size. Aborting."); - } - - skipSize = dis.readFloat(); - if (this.skipSize <= 0) { - throw new RuntimeException("File [" + fileName + "] has a negative number for window size. Aborting."); - } - - /* Read the number of units */ - numberOfUnits = dis.readInt(); - if (numberOfUnits < 0) { - throw new RuntimeException("File [" + fileName + "] has a negative number of units. Aborting."); - } - - contours = new double[numberOfUnits][]; - coeffs = new double[numberOfUnits][]; - voiced = new boolean[numberOfUnits][]; - - for (int i = 0; i < numberOfUnits; i++) { - - // read polynomial coeffs - int polyOrder = dis.readInt(); - coeffs[i] = new double[polyOrder]; - for (int j = 0; j < polyOrder; j++) { - coeffs[i][j] = dis.readFloat(); - } - - // read f0 contour - int f0ContourLength = dis.readInt(); - contours[i] = new double[f0ContourLength]; - - voiced[i] = new boolean[f0ContourLength]; - Arrays.fill(voiced[i], false); - for (int j = 0; j < f0ContourLength; j++) { - contours[i][j] = dis.readFloat(); - if (contours[i][j] > 0) { - voiced[i][j] = true; - } - } - } - } catch (IOException e) { - throw new RuntimeException("Reading the Mary header from file [" + fileName + "] failed.", e); - } - - } - - /*****************/ - /* OTHER METHODS */ - /*****************/ - - /** - * Get the number of units in the file. - * - * @return The number of units. - */ - public int getNumberOfUnits() { - return (numberOfUnits); - } - - /** - * get an intonation contour - * - * @param unitIndexNumber - * @return - */ - public double[] getContour(int unitIndexNumber) { - return this.contours[unitIndexNumber]; - } - - public boolean[] getVoicings(int unitIndexNumber) { - return this.voiced[unitIndexNumber]; - } - - public double getWindowSizeInSeconds() { - return this.windowSize; - } - - public double getSkipSizeInSeconds() { - return this.skipSize; - } - - /** - * get an intonation polynomial coeffs - * - * @param unitIndexNumber - * @return - */ - public double[] getIntonationCoeffs(int unitIndexNumber) { - return this.coeffs[unitIndexNumber]; - } - - public static void main(String[] args) throws Exception { - String fileName = "/home/sathish/Work/phd/voices/listener/vocalizations/timelines/vocalization_intonation_timeline.mry"; - VocalizationIntonationReader bcUfr = new VocalizationIntonationReader(fileName); - // bcUfr.load(fileName); - } -} diff --git a/marytts-runtime/src/main/java/marytts/vocalizations/VocalizationSelector.java b/marytts-runtime/src/main/java/marytts/vocalizations/VocalizationSelector.java deleted file mode 100644 index b034358d9d..0000000000 --- a/marytts-runtime/src/main/java/marytts/vocalizations/VocalizationSelector.java +++ /dev/null @@ -1,639 +0,0 @@ -/** - * Copyright 2010 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.vocalizations; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileReader; -import java.io.IOException; -import java.text.DecimalFormat; -import java.util.Arrays; -import java.util.List; - -import marytts.exceptions.MaryConfigurationException; -import marytts.features.FeatureDefinition; -import marytts.features.FeatureVector; -import marytts.modules.synthesis.Voice; -import marytts.server.MaryProperties; -import marytts.unitselection.data.Unit; -import marytts.unitselection.select.Target; -import marytts.unitselection.select.VocalizationFFRTargetCostFunction; -import marytts.util.MaryUtils; -import marytts.util.math.MathUtils; -import marytts.util.math.Polynomial; - -import org.apache.log4j.Level; -import org.apache.log4j.Logger; -import org.w3c.dom.Element; - -/** - * Select suitable vocalization for a given target using a cost function - * - * @author sathish - * - */ -public class VocalizationSelector { - - protected VocalizationFeatureFileReader featureFileReader; - protected FeatureDefinition featureDefinition; - protected FeatureDefinition f0FeatureDefinition; - protected VocalizationIntonationReader vIntonationReader; - protected VocalizationUnitFileReader unitFileReader; - protected VocalizationFFRTargetCostFunction vffrtUnitCostFunction = null; - protected VocalizationFFRTargetCostFunction vffrtContourCostFunction = null; - protected boolean f0ContourImposeSupport; - protected boolean usePrecondition; - protected double contourCostWeight; - private DecimalFormat df; - - protected int noOfSuitableUnits = 1; - - protected Logger logger = MaryUtils.getLogger("Vocalization Selector"); - - public VocalizationSelector(Voice voice) throws MaryConfigurationException { - - String unitFileName = MaryProperties.getFilename("voice." + voice.getName() + ".vocalization.unitfile"); - String featureFile = MaryProperties.getFilename("voice." + voice.getName() + ".vocalization.featurefile"); - String featureDefinitionFile = MaryProperties.getFilename("voice." + voice.getName() - + ".vocalization.featureDefinitionFile"); - f0ContourImposeSupport = MaryProperties.getBoolean("voice." + voice.getName() + ".f0ContourImposeSupport", false); - df = new DecimalFormat("##.##"); - try { - BufferedReader fDBufferedReader = new BufferedReader(new FileReader(new File(featureDefinitionFile))); - this.featureDefinition = new FeatureDefinition(fDBufferedReader, true); - this.featureFileReader = new VocalizationFeatureFileReader(featureFile); - vffrtUnitCostFunction = new VocalizationFFRTargetCostFunction(this.featureFileReader, this.featureDefinition); - unitFileReader = new VocalizationUnitFileReader(unitFileName); - - if (this.featureFileReader.getNumberOfUnits() != this.unitFileReader.getNumberOfUnits()) { - throw new MaryConfigurationException("Feature file reader and unit file reader is not aligned properly"); - } - - if (this.f0ContourImposeSupport) { - String intonationFDFile = MaryProperties.getFilename("voice." + voice.getName() - + ".vocalization.intonation.featureDefinitionFile"); - String intonationFile = MaryProperties.getFilename("voice." + voice.getName() + ".vocalization.intonationfile"); - usePrecondition = MaryProperties.getBoolean("voice." + voice.getName() + ".vocalization.usePrecondition", false); - contourCostWeight = (new Double(MaryProperties.getProperty("voice." + voice.getName() - + ".vocalization.contourCostWeight", "0.5"))).doubleValue(); - if (contourCostWeight < 0 || contourCostWeight > 1.0) { - throw new MaryConfigurationException("contourCostWeight should be between 0 and 1"); - } - BufferedReader f0FDBufferedReader = new BufferedReader(new FileReader(new File(intonationFDFile))); - f0FeatureDefinition = new FeatureDefinition(f0FDBufferedReader, true); - vIntonationReader = new VocalizationIntonationReader(intonationFile); - noOfSuitableUnits = MaryProperties.getInteger("voice." + voice.getName() - + ".vocalization.intonation.numberOfSuitableUnits"); - vffrtContourCostFunction = new VocalizationFFRTargetCostFunction(this.featureFileReader, this.f0FeatureDefinition); - } - } catch (IOException e) { - throw new MaryConfigurationException("Problem loading vocalization files for voice ", e); - } - } - - /** - * Get feature definition used to select suitable candidate - * - * @return Feature Definition - */ - public FeatureDefinition getFeatureDefinition() { - return this.featureDefinition; - } - - /** - * Get best candidate pair to impose F0 contour on other - * - * @param domElement - * xml request for vocalization - * @return SourceTargetPair best candidate pair - */ - public SourceTargetPair getBestCandidatePairtoImposeF0(Element domElement) { - - VocalizationCandidate[] vCosts = getBestMatchingCandidates(domElement); - VocalizationCandidate[] vIntonationCosts = getBestIntonationCandidates(domElement); - - int noOfSuitableF0Units; - if (usePrecondition) { - noOfSuitableF0Units = getNumberContoursAboveThreshold(vCosts, vIntonationCosts); - } else { - noOfSuitableF0Units = noOfSuitableUnits; - } - - if (noOfSuitableF0Units == 0) { - return new SourceTargetPair(vCosts[0].unitIndex, vCosts[0].unitIndex, 0); - } - - VocalizationCandidate[] suitableCandidates = new VocalizationCandidate[noOfSuitableUnits]; - System.arraycopy(vCosts, 0, suitableCandidates, 0, noOfSuitableUnits); - VocalizationCandidate[] suitableF0Candidates = new VocalizationCandidate[noOfSuitableF0Units]; - System.arraycopy(vIntonationCosts, 0, suitableF0Candidates, 0, noOfSuitableF0Units); - - Target targetUnit = createTarget(domElement); - if (logger.getEffectiveLevel().equals(Level.DEBUG)) { - debugLogCandidates(targetUnit, suitableCandidates, suitableF0Candidates); - } - - SourceTargetPair[] sortedImposeF0Data = vocalizationF0DistanceComputer(suitableCandidates, suitableF0Candidates, - domElement); - - return sortedImposeF0Data[0]; - } - - /** - * compute a threshold according to precondition and get number of contours above computed threshold - * - * formula : for all CC(j) < CCmax where j1, j2, j3 ... are contour candidates CCmax (threshold) = min (CC(i1), CC(i2), - * CC(i3)....) where i1, i2, i3 .. are unit candidates - * - * @param vCosts - * VocalizationCandidates - * @param vIntonationCosts - * VocalizationF0Candidates - * @return int number of contours above computed threshold - */ - private int getNumberContoursAboveThreshold(VocalizationCandidate[] vCosts, VocalizationCandidate[] vIntonationCosts) { - - // get minimum cc cost for all units - double[] costs = new double[noOfSuitableUnits]; - for (int i = 0; i < costs.length; i++) { - int unitIndex = vCosts[i].unitIndex; - VocalizationCandidate contourCandidate = getCandidateByIndex(vIntonationCosts, unitIndex); - if (contourCandidate != null) { - costs[i] = contourCandidate.cost; - } else { - costs[i] = Double.MAX_VALUE; - } - } - - double threshold = MathUtils.min(costs); - int contourSetSize = 0; - for (int i = 0; i < vIntonationCosts.length; i++) { - if (vIntonationCosts[i].cost < threshold) { - contourSetSize = i + 1; - } else { - break; - } - } - - return contourSetSize; - } - - /** - * search candidate by unitindex - * - * @param vIntonationCosts - * VocalizationF0Candidates - * @param unitIndex - * unitindex - * @return VocalizationCandidate of given index - */ - private VocalizationCandidate getCandidateByIndex(VocalizationCandidate[] vIntonationCosts, int unitIndex) { - - for (int i = 0; i < vIntonationCosts.length; i++) { - if (vIntonationCosts[i].unitIndex == unitIndex) { - return vIntonationCosts[i]; - } - } - return null; - } - - /** - * polynomial distance computer between two units - * - * @param suitableCandidates - * vocalization candidates - * @param suitableF0Candidates - * intonation candidates - * @return an array of candidate pairs - */ - private SourceTargetPair[] vocalizationF0DistanceComputer(VocalizationCandidate[] suitableCandidates, - VocalizationCandidate[] suitableF0Candidates, Element domElement) { - - int noPossibleImpositions = suitableCandidates.length * suitableF0Candidates.length; - SourceTargetPair[] imposeF0Data = new SourceTargetPair[noPossibleImpositions]; - int count = 0; - - for (int i = 0; i < suitableCandidates.length; i++) { - for (int j = 0; j < suitableF0Candidates.length; j++) { - int sourceIndex = suitableCandidates[i].unitIndex; - int targetIndex = suitableF0Candidates[j].unitIndex; - - double contourCost = getContourCostDistance(sourceIndex, targetIndex); - double mergeCost = getMergeCost(sourceIndex, targetIndex, domElement); - double cost = (contourCost * contourCostWeight) + (mergeCost * (1 - contourCostWeight)); - logger.debug("Unit Index " + sourceIndex + " & Contour Index " + targetIndex + " :: Countour cost: " - + df.format(contourCost) + " + Merge Cost: " + df.format(mergeCost) + " --> TotalCost: " - + df.format(cost)); - imposeF0Data[count++] = new SourceTargetPair(sourceIndex, targetIndex, cost); - } - } - - Arrays.sort(imposeF0Data); - - return imposeF0Data; - } - - /** - * Compute mergeCost for unit and contour candidates Formula = segmentalformCost(u(i)) + intonationCost(c(i)) + - * voiceQualityCost(u(i)) + 0.5 * (meaningCost(u(i)) + meaningCost(c(i)) ) - * - * @param sourceIndex - * unit index - * @param targetIndex - * unit index - * @return double merge cost - */ - private double getMergeCost(int sourceIndex, int targetIndex, Element domElement) { - - Target targetUnit = createTarget(domElement); - Target targetContour = createIntonationTarget(domElement); - - Unit unitCandidate = this.unitFileReader.getUnit(sourceIndex); - Unit contourCandidate = this.unitFileReader.getUnit(targetIndex); - - // unit features - double segmentalCost = vffrtUnitCostFunction.featureCost(targetUnit, unitCandidate, "name"); - double voiceQualityCost = vffrtUnitCostFunction.featureCost(targetUnit, unitCandidate, "voicequality"); - double meaningUnitCost = 0; - String[] meaningFeatureNames = vffrtUnitCostFunction.getFeatureDefinition().getContinuousFeatureNameArray(); - for (int i = 0; i < meaningFeatureNames.length; i++) { - meaningUnitCost += vffrtUnitCostFunction.featureCost(targetUnit, unitCandidate, meaningFeatureNames[i]); - } - - // contour features - double intonationCost = this.vffrtContourCostFunction.featureCost(targetContour, contourCandidate, "intonation"); - double meaningContourCost = 0; - String[] meaningContourFeatures = vffrtContourCostFunction.getFeatureDefinition().getContinuousFeatureNameArray(); - for (int i = 0; i < meaningContourFeatures.length; i++) { - meaningContourCost += vffrtContourCostFunction - .featureCost(targetContour, contourCandidate, meaningContourFeatures[i]); - } - - double mergeCost = segmentalCost + voiceQualityCost + intonationCost + 0.5 * (meaningUnitCost + meaningContourCost); - - return mergeCost; - } - - /** - * compute contour distance (polynomial distance) - * - * @param sourceIndex - * unit index - * @param targetIndex - * unit index - * @return polynomial distance measure - */ - private double getContourCostDistance(int sourceIndex, int targetIndex) { - double distance; - if (targetIndex == sourceIndex) { - distance = 0; - } else { - double[] targetCoeffs = vIntonationReader.getIntonationCoeffs(targetIndex); - double[] sourceCoeffs = vIntonationReader.getIntonationCoeffs(sourceIndex); - if (targetCoeffs != null && sourceCoeffs != null && targetCoeffs.length == sourceCoeffs.length) { - distance = Polynomial.polynomialDistance(sourceCoeffs, targetCoeffs); - } else { - distance = Double.MAX_VALUE; - } - } - return distance; - } - - /** - * get a best matching candidate for a given target - * - * @param domElement - * xml request for vocalization - * @return unit index of best matching candidate - */ - public int getBestMatchingCandidate(Element domElement) { - - Target targetUnit = createTarget(domElement); - int numberUnits = this.unitFileReader.getNumberOfUnits(); - double minCost = Double.MAX_VALUE; - int index = 0; - for (int i = 0; i < numberUnits; i++) { - Unit singleUnit = this.unitFileReader.getUnit(i); - double cost = vffrtUnitCostFunction.cost(targetUnit, singleUnit); - if (cost < minCost) { - minCost = cost; - index = i; - } - } - - return index; - } - - /** - * get a array of best candidates sorted according to cost - * - * @param domElement - * xml request for vocalization - * @return an array of best vocalization candidates - */ - public VocalizationCandidate[] getBestMatchingCandidates(Element domElement) { - // FeatureDefinition featDef = this.featureFileReader.getFeatureDefinition(); - Target targetUnit = createTarget(domElement); - int numberUnits = this.unitFileReader.getNumberOfUnits(); - VocalizationCandidate[] vocalizationCandidates = new VocalizationCandidate[numberUnits]; - for (int i = 0; i < numberUnits; i++) { - Unit singleUnit = this.unitFileReader.getUnit(i); - double cost = vffrtUnitCostFunction.cost(targetUnit, singleUnit); - vocalizationCandidates[i] = new VocalizationCandidate(i, cost); - } - Arrays.sort(vocalizationCandidates); - return vocalizationCandidates; - } - - /** - * Debug messages for selected candidates - * - * @param targetUnit - * target unit - * @param suitableCandidates - * suitable vocalization candidates - * @param suitableF0Candidates - * suitable intonation candidates - */ - private void debugLogCandidates(Target targetUnit, VocalizationCandidate[] suitableCandidates, - VocalizationCandidate[] suitableF0Candidates) { - FeatureVector targetFeatures = targetUnit.getFeatureVector(); - FeatureDefinition fd = featureFileReader.getFeatureDefinition(); - int fiName = fd.getFeatureIndex("name"); - int fiIntonation = fd.getFeatureIndex("intonation"); - int fiVQ = fd.getFeatureIndex("voicequality"); - for (int i = 0; i < suitableCandidates.length; i++) { - int unitIndex = suitableCandidates[i].unitIndex; - double unitCost = suitableCandidates[i].cost; - FeatureVector fv = featureFileReader.getFeatureVector(unitIndex); - StringBuilder sb = new StringBuilder(); - sb.append("Candidate ").append(i).append(": ").append(unitIndex).append(" ( " + unitCost + " ) ").append(" -- "); - byte bName = fv.getByteFeature(fiName); - if (fv.getByteFeature(fiName) != 0 && targetFeatures.getByteFeature(fiName) != 0) { - sb.append(" ").append(fv.getFeatureAsString(fiName, fd)); - } - if (fv.getByteFeature(fiVQ) != 0 && targetFeatures.getByteFeature(fiName) != 0) { - sb.append(" ").append(fv.getFeatureAsString(fiVQ, fd)); - } - if (fv.getByteFeature(fiIntonation) != 0 && targetFeatures.getByteFeature(fiIntonation) != 0) { - sb.append(" ").append(fv.getFeatureAsString(fiIntonation, fd)); - } - for (int j = 0; j < targetFeatures.getLength(); j++) { - if (targetFeatures.isContinuousFeature(j) && !Float.isNaN((Float) targetFeatures.getFeature(j)) - && !Float.isNaN((Float) fv.getFeature(j))) { - String featureName = fd.getFeatureName(j); - sb.append(" ").append(featureName).append("=").append(fv.getFeature(j)); - } - } - logger.debug(sb.toString()); - } - for (int i = 0; i < suitableF0Candidates.length; i++) { - int unitIndex = suitableF0Candidates[i].unitIndex; - double unitCost = suitableF0Candidates[i].cost; - FeatureVector fv = featureFileReader.getFeatureVector(unitIndex); - StringBuilder sb = new StringBuilder(); - sb.append("F0 Candidate ").append(i).append(": ").append(unitIndex).append(" ( " + unitCost + " ) ").append(" -- "); - byte bName = fv.getByteFeature(fiName); - if (fv.getByteFeature(fiName) != 0 && targetFeatures.getByteFeature(fiName) != 0) { - sb.append(" ").append(fv.getFeatureAsString(fiName, fd)); - } - if (fv.getByteFeature(fiVQ) != 0 && targetFeatures.getByteFeature(fiName) != 0) { - sb.append(" ").append(fv.getFeatureAsString(fiVQ, fd)); - } - if (fv.getByteFeature(fiIntonation) != 0 && targetFeatures.getByteFeature(fiIntonation) != 0) { - sb.append(" ").append(fv.getFeatureAsString(fiIntonation, fd)); - } - for (int j = 0; j < targetFeatures.getLength(); j++) { - if (targetFeatures.isContinuousFeature(j) && !Float.isNaN((Float) targetFeatures.getFeature(j)) - && !Float.isNaN((Float) fv.getFeature(j))) { - String featureName = fd.getFeatureName(j); - sb.append(" ").append(featureName).append("=").append(fv.getFeature(j)); - } - } - logger.debug(sb.toString()); - } - } - - /** - * get a array of best candidates sorted according to cost (cost computed on f0_feature_definition features only) - * - * @param domElement - * xml request for vocalization - * @return VocalizationCandidate[] a array of best candidates - */ - private VocalizationCandidate[] getBestIntonationCandidates(Element domElement) { - - Target targetUnit = createIntonationTarget(domElement); - int numberUnits = this.unitFileReader.getNumberOfUnits(); - VocalizationCandidate[] vocalizationCandidates = new VocalizationCandidate[numberUnits]; - for (int i = 0; i < numberUnits; i++) { - Unit singleUnit = this.unitFileReader.getUnit(i); - double cost = vffrtContourCostFunction.cost(targetUnit, singleUnit); - vocalizationCandidates[i] = new VocalizationCandidate(i, cost); - } - Arrays.sort(vocalizationCandidates); - return vocalizationCandidates; - } - - /** - * create target from XML request - * - * @param domElement - * xml request for vocalization - * @return Target target represents xml request - */ - private Target createTarget(Element domElement) { - - // FeatureDefinition featDef = this.featureFileReader.getFeatureDefinition(); - FeatureDefinition featDef = this.featureDefinition; - int numFeatures = featDef.getNumberOfFeatures(); - int numByteFeatures = featDef.getNumberOfByteFeatures(); - int numShortFeatures = featDef.getNumberOfShortFeatures(); - int numContiniousFeatures = featDef.getNumberOfContinuousFeatures(); - byte[] byteFeatures = new byte[numByteFeatures]; - short[] shortFeatures = new short[numShortFeatures]; - float[] floatFeatures = new float[numContiniousFeatures]; - int byteCount = 0; - int shortCount = 0; - int floatCount = 0; - - for (int i = 0; i < numFeatures; i++) { - - String featName = featDef.getFeatureName(i); - String featValue = "0"; - - if (featDef.isByteFeature(featName) || featDef.isShortFeature(featName)) { - if (domElement.hasAttribute(featName)) { - featValue = domElement.getAttribute(featName); - } - - boolean hasFeature = featDef.hasFeatureValue(featName, featValue); - if (!hasFeature) - featValue = "0"; - - if (featDef.isByteFeature(i)) { - byteFeatures[byteCount++] = featDef.getFeatureValueAsByte(i, featValue); - } else if (featDef.isShortFeature(i)) { - shortFeatures[shortCount++] = featDef.getFeatureValueAsShort(i, featValue); - } - } else { - if (domElement.hasAttribute("meaning")) { - featValue = domElement.getAttribute("meaning"); - } - // float contFeature = getMeaningScaleValue ( featName, featValue ); - floatFeatures[floatCount++] = getMeaningScaleValue(featName, featValue); - } - } - - FeatureVector newFV = featDef.toFeatureVector(0, byteFeatures, shortFeatures, floatFeatures); - - String name = "0"; - if (domElement.hasAttribute("name")) { - name = domElement.getAttribute("name"); - } - - Target newTarget = new Target(name, domElement); - newTarget.setFeatureVector(newFV); - - return newTarget; - } - - /** - * create F0 target from XML request - * - * @param domElement - * xml request for intonation - * @return Target target represents xml request - */ - private Target createIntonationTarget(Element domElement) { - - // FeatureDefinition featDef = this.featureFileReader.getFeatureDefinition(); - FeatureDefinition featDef = this.f0FeatureDefinition; - int numFeatures = featDef.getNumberOfFeatures(); - int numByteFeatures = featDef.getNumberOfByteFeatures(); - int numShortFeatures = featDef.getNumberOfShortFeatures(); - int numContiniousFeatures = featDef.getNumberOfContinuousFeatures(); - byte[] byteFeatures = new byte[numByteFeatures]; - short[] shortFeatures = new short[numShortFeatures]; - float[] floatFeatures = new float[numContiniousFeatures]; - int byteCount = 0; - int shortCount = 0; - int floatCount = 0; - - for (int i = 0; i < numFeatures; i++) { - - String featName = featDef.getFeatureName(i); - String featValue = "0"; - - if (featDef.isByteFeature(featName) || featDef.isShortFeature(featName)) { - if (domElement.hasAttribute(featName)) { - featValue = domElement.getAttribute(featName); - } - - boolean hasFeature = featDef.hasFeatureValue(featName, featValue); - if (!hasFeature) - featValue = "0"; - - if (featDef.isByteFeature(i)) { - byteFeatures[byteCount++] = featDef.getFeatureValueAsByte(i, featValue); - } else if (featDef.isShortFeature(i)) { - shortFeatures[shortCount++] = featDef.getFeatureValueAsShort(i, featValue); - } - } else { - if (domElement.hasAttribute("meaning")) { - featValue = domElement.getAttribute("meaning"); - } - // float contFeature = getMeaningScaleValue ( featName, featValue ); - floatFeatures[floatCount++] = getMeaningScaleValue(featName, featValue); - } - } - - FeatureVector newFV = featDef.toFeatureVector(0, byteFeatures, shortFeatures, floatFeatures); - - String name = "0"; - if (domElement.hasAttribute("name")) { - name = domElement.getAttribute("name"); - } - - Target newTarget = new Target(name, domElement); - newTarget.setFeatureVector(newFV); - - return newTarget; - } - - /** - * get value on meaning scale as a float value - * - * @param featureName - * feature names - * @param meaningAttribute - * meaning attribute - * @return a float value for a meaning feature - */ - private float getMeaningScaleValue(String featureName, String meaningAttribute) { - - String[] categories = meaningAttribute.split("\\s+"); - List categoriesList = Arrays.asList(categories); - - if ("anger".equals(featureName) && categoriesList.contains("anger")) { - return 5; - } else if ("sadness".equals(featureName) && categoriesList.contains("sadness")) { - return 5; - } else if ("amusement".equals(featureName) && categoriesList.contains("amusement")) { - return 5; - } else if ("happiness".equals(featureName) && categoriesList.contains("happiness")) { - return 5; - } else if ("contempt".equals(featureName) && categoriesList.contains("contempt")) { - return 5; - } else if ("certain".equals(featureName) && categoriesList.contains("uncertain")) { - return -2; - } else if ("certain".equals(featureName) && categoriesList.contains("certain")) { - return 2; - } else if ("agreeing".equals(featureName) && categoriesList.contains("disagreeing")) { - return -2; - } else if ("agreeing".equals(featureName) && categoriesList.contains("agreeing")) { - return 2; - } else if ("interested".equals(featureName) && categoriesList.contains("uninterested")) { - return -2; - } else if ("interested".equals(featureName) && categoriesList.contains("interested")) { - return 2; - } else if ("anticipation".equals(featureName) && categoriesList.contains("low-anticipation")) { - return -2; - } else if ("anticipation".equals(featureName) && categoriesList.contains("anticipation")) { - return 2; - } else if ("anticipation".equals(featureName) && categoriesList.contains("high-anticipation")) { - return 2; - } else if ("solidarity".equals(featureName) && categoriesList.contains("solidarity")) { - return 5; - } else if ("solidarity".equals(featureName) && categoriesList.contains("low-solidarity")) { - return 1; - } else if ("solidarity".equals(featureName) && categoriesList.contains("high-solidarity")) { - return 5; - } else if ("antagonism".equals(featureName) && categoriesList.contains("antagonism")) { - return 5; - } else if ("antagonism".equals(featureName) && categoriesList.contains("high-antagonism")) { - return 5; - } else if ("antagonism".equals(featureName) && categoriesList.contains("low-antagonism")) { - return 1; - } - - return Float.NaN; - } -} diff --git a/marytts-runtime/src/main/java/marytts/vocalizations/VocalizationSynthesisTechnology.java b/marytts-runtime/src/main/java/marytts/vocalizations/VocalizationSynthesisTechnology.java deleted file mode 100644 index 3b4afbc72d..0000000000 --- a/marytts-runtime/src/main/java/marytts/vocalizations/VocalizationSynthesisTechnology.java +++ /dev/null @@ -1,77 +0,0 @@ -/** - * Copyright 2000-2006 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.vocalizations; - -import javax.sound.sampled.AudioFileFormat; -import javax.sound.sampled.AudioInputStream; - -import marytts.exceptions.SynthesisException; - -/** - * An abstract class for vocalization syntehsis technology - * - * @author Sathish Pammi - */ - -public abstract class VocalizationSynthesisTechnology { - - /** - * Synthesize given vocalization - * - * @param unitIndex - * unit index - * @param aft - * audio file format - * @return AudioInputStream of synthesized vocalization - * @throws SynthesisException - * if failed to synthesize vocalization - */ - public abstract AudioInputStream synthesize(int unitIndex, AudioFileFormat aft) throws SynthesisException; - - /** - * Re-synthesize given vocalization - * - * @param unitIndex - * unit index - * @param aft - * audio file format - * @return AudioInputStream of synthesized vocalization - * @throws SynthesisException - * if failed to synthesize vocalization - */ - public abstract AudioInputStream reSynthesize(int sourceIndex, AudioFileFormat aft) throws SynthesisException; - - /** - * Impose target intonation contour on given vocalization - * - * @param sourceIndex - * unit index of vocalization - * @param targetIndex - * unit index of target intonation - * @param aft - * aft audio file format - * @return AudioInputStream of synthesized vocalization - * @throws SynthesisException - * if failed to synthesize vocalization - */ - public abstract AudioInputStream synthesizeUsingImposedF0(int sourceIndex, int targetIndex, AudioFileFormat aft) - throws SynthesisException; - -} diff --git a/marytts-runtime/src/main/java/marytts/vocalizations/VocalizationSynthesizer.java b/marytts-runtime/src/main/java/marytts/vocalizations/VocalizationSynthesizer.java deleted file mode 100644 index a358561816..0000000000 --- a/marytts-runtime/src/main/java/marytts/vocalizations/VocalizationSynthesizer.java +++ /dev/null @@ -1,305 +0,0 @@ -/** - * Copyright 2000-2006 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.vocalizations; - -import java.io.IOException; - -import javax.sound.sampled.AudioFileFormat; -import javax.sound.sampled.AudioInputStream; - -import marytts.datatypes.MaryXML; -import marytts.exceptions.MaryConfigurationException; -import marytts.exceptions.SynthesisException; -import marytts.features.FeatureDefinition; -import marytts.modules.synthesis.Voice; -import marytts.server.MaryProperties; -import marytts.unitselection.data.Unit; -import marytts.util.MaryUtils; - -import org.apache.log4j.Logger; -import org.w3c.dom.Element; - -/** - * The vocalization synthesis module. - * - * @author Sathish Pammi - */ - -public class VocalizationSynthesizer { - - protected VocalizationSynthesisTechnology vSynthesizer; - protected VocalizationSelector vSelector; - protected VocalizationUnitFileReader unitFileReader; - protected boolean f0ContourImposeSupport; - - protected Logger logger = MaryUtils.getLogger("Vocalization Synthesizer"); - - public VocalizationSynthesizer(Voice voice) throws MaryConfigurationException { - - if (!voice.hasVocalizationSupport()) { - throw new MaryConfigurationException("This voice " + voice.toString() + " doesn't support synthesis of vocalizations"); - } - - String unitFileName = MaryProperties.getFilename("voice." + voice.getName() + ".vocalization.unitfile"); - - try { - this.unitFileReader = new VocalizationUnitFileReader(unitFileName); - } catch (IOException e) { - throw new MaryConfigurationException("can't read unit file"); - } - - String intonationFile = MaryProperties.getFilename("voice." + voice.getName() + ".vocalization.intonationfile"); - String technology = MaryProperties.getProperty("voice." + voice.getName() + ".vocalization.synthesisTechnology", - "fdpsola"); - f0ContourImposeSupport = MaryProperties.getBoolean("voice." + voice.getName() + ".f0ContourImposeSupport", false); - - if ("fdpsola".equals(technology)) { - String timelineFile = MaryProperties.getFilename("voice." + voice.getName() + ".vocalization.timeline"); - vSynthesizer = new FDPSOLASynthesisTechnology(timelineFile, unitFileName, intonationFile, f0ContourImposeSupport); - } else if ("mlsa".equals(technology)) { - boolean imposePolynomialContour = MaryProperties.getBoolean("voice." + voice.getName() - + ".vocalization.imposePolynomialContour", true); - String mlsaFeatureFile = MaryProperties.getFilename("voice." + voice.getName() + ".vocalization.mlsafeaturefile"); - String mixedExcitationFilter = MaryProperties.getFilename("voice." + voice.getName() - + ".vocalization.mixedexcitationfilter"); - vSynthesizer = new MLSASynthesisTechnology(mlsaFeatureFile, intonationFile, mixedExcitationFilter, - imposePolynomialContour); - } else if ("hnm".equals(technology)) { - String timelineFile = MaryProperties.getFilename("voice." + voice.getName() + ".vocalization.timeline"); - String hnmFeatureFile = MaryProperties.getFilename("voice." + voice.getName() + ".vocalization.hnmfeaturefile"); - vSynthesizer = new HNMSynthesisTechnology(timelineFile, unitFileName, hnmFeatureFile, intonationFile, - f0ContourImposeSupport); - } else { - throw new MaryConfigurationException("the property 'voice." + voice.getName() - + ".vocalization.synthesisTechnology' should be one among 'hnm', 'mlsa' and 'fdpsola'"); - } - - this.vSelector = new VocalizationSelector(voice); - } - - /** - * Handle a request for synthesis of vocalization - * - * @param voice - * the selected voice - * @param aft - * AudioFileFormat of the output AudioInputStream - * @param domElement - * target xml element ('vocalization' element) - * @return AudioInputStream of requested vocalization it returns null if the voice doesn't support synthesis of vocalizations - * @throws IllegalArgumentException - * if domElement contains 'variant' attribute value is greater than available number of vocalizations - */ - public AudioInputStream synthesize(Voice voice, AudioFileFormat aft, Element domElement) throws Exception { - - if (!voice.hasVocalizationSupport()) - return null; - - if (domElement.hasAttribute("variant")) { - return synthesizeVariant(aft, domElement); - } - - if (f0ContourImposeSupport) { - return synthesizeImposedIntonation(aft, domElement); - } - - return synthesizeVocalization(aft, domElement); - } - - /** - * Synthesize a "variant" vocalization - * - * @param aft - * AudioFileFormat of the output AudioInputStream - * @param domElement - * target 'vocalization' xml element - * @return AudioInputStream of requested vocalization - * @throws SynthesisException - * if it can't synthesize vocalization - * @throws IllegalArgumentException - * if domElement contains 'variant' attribute value is greater than available number of vocalizations - */ - private AudioInputStream synthesizeVariant(AudioFileFormat aft, Element domElement) throws SynthesisException { - - int numberOfBackChannels = unitFileReader.getNumberOfUnits(); - int backchannelNumber = 0; - - if (domElement.hasAttribute("variant")) { - backchannelNumber = Integer.parseInt(domElement.getAttribute("variant")); - } - - if (backchannelNumber >= numberOfBackChannels) { - throw new IllegalArgumentException("This voice has " + numberOfBackChannels - + " backchannels only. so it doesn't support unit number " + backchannelNumber); - } - - return synthesizeSelectedVocalization(backchannelNumber, aft, domElement); - } - - /** - * Synthesize a vocalization which fits better for given target - * - * @param aft - * AudioFileFormat of the output AudioInputStream - * @param domElement - * target 'vocalization' xml element - * @return AudioInputStream output audio - * @throws SynthesisException - * if it can't synthesize vocalization - */ - private AudioInputStream synthesizeVocalization(AudioFileFormat aft, Element domElement) throws SynthesisException { - - int numberOfBackChannels = unitFileReader.getNumberOfUnits(); - int backchannelNumber = vSelector.getBestMatchingCandidate(domElement); - // here it is a bug, if getBestMatchingCandidate select a backchannelNumber greater than numberOfBackChannels - assert backchannelNumber < numberOfBackChannels : "This voice has " + numberOfBackChannels - + " backchannels only. so it doesn't support unit number " + backchannelNumber; - - return synthesizeSelectedVocalization(backchannelNumber, aft, domElement); - } - - /** - * Synthesize a vocalization which fits better for given target, in addition, impose intonation from closest best vocalization - * according to given feature definition for intonation selection - * - * @param aft - * AudioFileFormat of the output AudioInputStream - * @param domElement - * target 'vocalization' xml element - * @return AudioInputStream output audio - * @throws SynthesisException - * if it can't synthesize vocalization - */ - private AudioInputStream synthesizeImposedIntonation(AudioFileFormat aft, Element domElement) throws SynthesisException { - - SourceTargetPair imposeF0Data = vSelector.getBestCandidatePairtoImposeF0(domElement); - int targetIndex = imposeF0Data.getTargetUnitIndex(); - int sourceIndex = imposeF0Data.getSourceUnitIndex(); - - logger.debug("Synthesizing candidate " + sourceIndex + " with intonation contour " + targetIndex); - - if (targetIndex == sourceIndex) { - return synthesizeSelectedVocalization(sourceIndex, aft, domElement); - } - - return imposeF0ContourOnVocalization(sourceIndex, targetIndex, aft, domElement); - } - - /** - * Impose a target f0 contour onto a (source) unit - * - * @param sourceIndex - * unit index of segmentalform unit - * @param targetIndex - * unit index of target f0 contour - * @param aft - * AudioFileFormat of the output AudioInputStream - * @param domElement - * target 'vocalization' xml element - * @return AudioInputStream of requested vocalization - * @throws SynthesisException - * if no data can be read at the given target time or if audio processing fails - */ - private AudioInputStream imposeF0ContourOnVocalization(int sourceIndex, int targetIndex, AudioFileFormat aft, - Element domElement) throws SynthesisException { - - int numberOfBackChannels = unitFileReader.getNumberOfUnits(); - - if (targetIndex >= numberOfBackChannels) { - throw new IllegalArgumentException("This voice has " + numberOfBackChannels - + " backchannels only. so it doesn't support unit number " + targetIndex); - } - - if (sourceIndex >= numberOfBackChannels) { - throw new IllegalArgumentException("This voice has " + numberOfBackChannels - + " backchannels only. so it doesn't support unit number " + sourceIndex); - } - - VocalizationUnit bUnit = unitFileReader.getUnit(sourceIndex); - Unit[] units = bUnit.getUnits(); - String[] unitNames = bUnit.getUnitNames(); - long endTime = 0l; - for (int i = 0; i < units.length; i++) { - int unitDuration = units[i].duration * 1000 / unitFileReader.getSampleRate(); - endTime += unitDuration; - Element element = MaryXML.createElement(domElement.getOwnerDocument(), MaryXML.PHONE); - element.setAttribute("d", Integer.toString(unitDuration)); - element.setAttribute("end", Long.toString(endTime)); - element.setAttribute("p", unitNames[i]); - domElement.appendChild(element); - } - - return this.vSynthesizer.synthesizeUsingImposedF0(sourceIndex, targetIndex, aft); - } - - /** - * Synthesize a selected vocalization - * - * @param backchannelNumber - * unit index number - * @param aft - * AudioFileFormat of the output AudioInputStream - * @param domElement - * target 'vocalization' xml element - * @return AudioInputStream output audio - * @throws SynthesisException - * if it can't synthesize vocalization - * @throws IllegalArgumentException - * if given backchannelNumber > no. of available vocalizations - */ - private AudioInputStream synthesizeSelectedVocalization(int backchannelNumber, AudioFileFormat aft, Element domElement) - throws SynthesisException { - - int numberOfBackChannels = unitFileReader.getNumberOfUnits(); - if (backchannelNumber >= numberOfBackChannels) { - throw new IllegalArgumentException("This voice has " + numberOfBackChannels - + " backchannels only. so it doesn't support unit number " + backchannelNumber); - } - - VocalizationUnit bUnit = unitFileReader.getUnit(backchannelNumber); - Unit[] units = bUnit.getUnits(); - String[] unitNames = bUnit.getUnitNames(); - long endTime = 0l; - for (int i = 0; i < units.length; i++) { - int unitDuration = units[i].duration * 1000 / unitFileReader.getSampleRate(); - endTime += unitDuration; - Element element = MaryXML.createElement(domElement.getOwnerDocument(), MaryXML.PHONE); - element.setAttribute("d", Integer.toString(unitDuration)); - element.setAttribute("end", Long.toString(endTime)); - element.setAttribute("p", unitNames[i]); - domElement.appendChild(element); - } - - return this.vSynthesizer.synthesize(backchannelNumber, aft); - } - - /** - * List the possible vocalization names that are available for the given voice. These values can be used in the "name" - * attribute of the vocalization tag. - * - * @return an array of Strings, each string containing one unique vocalization name. - */ - public String[] listAvailableVocalizations() { - FeatureDefinition featureDefinition = vSelector.getFeatureDefinition(); - assert featureDefinition.hasFeature("name"); - int nameIndex = featureDefinition.getFeatureIndex("name"); - return featureDefinition.getPossibleValues(nameIndex); - } -} diff --git a/marytts-runtime/src/main/java/marytts/vocalizations/VocalizationUnit.java b/marytts-runtime/src/main/java/marytts/vocalizations/VocalizationUnit.java deleted file mode 100644 index fa17bf9dea..0000000000 --- a/marytts-runtime/src/main/java/marytts/vocalizations/VocalizationUnit.java +++ /dev/null @@ -1,59 +0,0 @@ -/** - * Copyright 2006 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.vocalizations; - -import marytts.unitselection.data.Unit; - -/** - * Representation of a unit from a unit database. This gives access to everything that is known about a given unit, including all - * sorts of features and the actual audio data. - * - * @author Sathish pammi - * - */ -public class VocalizationUnit extends marytts.unitselection.data.Unit { - protected Unit[] units; - protected String[] unitNames; - - public VocalizationUnit(long startTime, int duration, int index) { - super(startTime, duration, index); - } - - /** - * Set units - * - * @return - */ - public void setUnits(Unit[] units) { - this.units = units; - } - - public Unit[] getUnits() { - return this.units; - } - - public void setUnitNames(String[] unitNames) { - this.unitNames = unitNames; - } - - public String[] getUnitNames() { - return this.unitNames; - } -} diff --git a/marytts-runtime/src/main/java/marytts/vocalizations/VocalizationUnitFileReader.java b/marytts-runtime/src/main/java/marytts/vocalizations/VocalizationUnitFileReader.java deleted file mode 100644 index 990ca7462a..0000000000 --- a/marytts-runtime/src/main/java/marytts/vocalizations/VocalizationUnitFileReader.java +++ /dev/null @@ -1,240 +0,0 @@ -/** - * Portions Copyright 2006 DFKI GmbH. - * Portions Copyright 2001 Sun Microsystems, Inc. - * Portions Copyright 1999-2001 Language Technologies Institute, - * Carnegie Mellon University. - * All Rights Reserved. Use is subject to license terms. - * - * Permission is hereby granted, free of charge, to use and distribute - * this software and its documentation without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of this work, and to - * permit persons to whom this work is furnished to do so, subject to - * the following conditions: - * - * 1. The code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * 2. Any modifications must be clearly marked as such. - * 3. Original authors' names are not deleted. - * 4. The authors' names are not used to endorse or promote products - * derived from this software without specific prior written - * permission. - * - * DFKI GMBH AND THE CONTRIBUTORS TO THIS WORK DISCLAIM ALL WARRANTIES WITH - * REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL DFKI GMBH NOR THE - * CONTRIBUTORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL - * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR - * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS - * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF - * THIS SOFTWARE. - */ -package marytts.vocalizations; - -import java.io.BufferedInputStream; -import java.io.DataInputStream; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.IOException; - -import marytts.exceptions.MaryConfigurationException; -import marytts.unitselection.data.Unit; -import marytts.util.data.MaryHeader; - -/** - * Loads a unit file in memory and provides accessors to the start times and durations. - * - * @author sathish pammi - * - */ -public class VocalizationUnitFileReader { - - private MaryHeader hdr = null; - private int numberOfUnits = 0; - private int sampleRate = 0; - private VocalizationUnit[] backchannelUnits; - - /****************/ - /* CONSTRUCTORS */ - /****************/ - - /** - * Empty constructor; need to call load() separately. - * - * @see #load(String) - */ - public VocalizationUnitFileReader() { - } - - /** - * Create a unit file reader from the given unit file - * - * @param fileName - * the unit file to read - * @throws IOException - * if a problem occurs while reading - */ - public VocalizationUnitFileReader(String fileName) throws IOException, MaryConfigurationException { - load(fileName); - } - - /** - * Load the given unit file - * - * @param fileName - * the unit file to read - * @throws IOException - * if a problem occurs while reading - */ - public void load(String fileName) throws IOException, MaryConfigurationException { - /* Open the file */ - DataInputStream dis = null; - try { - dis = new DataInputStream(new BufferedInputStream(new FileInputStream(fileName))); - } catch (FileNotFoundException e) { - throw new RuntimeException("File [" + fileName + "] was not found."); - } - try { - /* Load the Mary header */ - hdr = new MaryHeader(dis); - if (hdr.getType() != MaryHeader.LISTENERUNITS) { - throw new RuntimeException("File [" + fileName + "] is not a valid Mary Units file."); - } - /* Read the number of units */ - numberOfUnits = dis.readInt(); - // System.out.println("No. of units : "+ numberOfUnits); - if (numberOfUnits < 0) { - throw new RuntimeException("File [" + fileName + "] has a negative number of units. Aborting."); - } - /* Read the sample rate */ - sampleRate = dis.readInt(); - // System.out.println("Samplerate : "+ sampleRate); - if (sampleRate < 0) { - throw new RuntimeException("File [" + fileName + "] has a negative number sample rate. Aborting."); - } - - backchannelUnits = new VocalizationUnit[numberOfUnits]; - - /* Read the start times and durations */ - for (int i = 0; i < numberOfUnits; i++) { - int noOfUnits = dis.readInt(); - // System.out.println("No. of Local Units : "+ noOfUnits); - Unit[] units = new Unit[noOfUnits]; - String[] unitNames = new String[noOfUnits]; - for (int j = 0; j < noOfUnits; j++) { - long startTime = dis.readLong(); - int duration = dis.readInt(); - // System.out.println("Local Unit Data : "+ startTime+" "+ duration+" "+ j); - units[j] = new Unit(startTime, duration, j); - int charArraySize = dis.readInt(); - char[] phoneChar = new char[charArraySize]; - for (int k = 0; k < charArraySize; k++) { - phoneChar[k] = dis.readChar(); - } - unitNames[j] = new String(phoneChar); - } - long startBCTime = units[0].startTime; - int bcDuration = (((int) units[noOfUnits - 1].startTime + units[noOfUnits - 1].duration) - (int) units[0].startTime); - backchannelUnits[i] = new VocalizationUnit(startBCTime, bcDuration, i); - backchannelUnits[i].setUnits(units); - backchannelUnits[i].setUnitNames(unitNames); - // System.out.println("BC UNIT START:"+backchannelUnits[i].getStart()); - // System.out.println("BC UNIT Duration:"+backchannelUnits[i].getDuration()); - } - } catch (IOException e) { - throw new RuntimeException("Reading the Mary header from file [" + fileName + "] failed.", e); - } - - } - - /*****************/ - /* OTHER METHODS */ - /*****************/ - - /** - * Get the number of units in the file. - * - * @return The number of units. - */ - public int getNumberOfUnits() { - return (numberOfUnits); - } - - /** - * Get the sample rate of the file. - * - * @return The sample rate, in Hz. - */ - public int getSampleRate() { - return (sampleRate); - } - - /** - * Return the unit number i. - * - * @param i - * The index of the considered unit. - * @return The considered unit. - */ - public VocalizationUnit getUnit(int i) { - return backchannelUnits[i]; - } - - /** - * Return an array of units from their indexes. - * - * @param i - * The indexes of the considered units. - * @return The array of considered units. - */ - public VocalizationUnit[] getUnit(int[] i) { - VocalizationUnit[] ret = new VocalizationUnit[i.length]; - for (int k = 0; k < i.length; k++) { - ret[k] = getUnit(i[k]); - } - return (ret); - } - - /** - * Return the unit following the given unit in the original database. - * - * @param u - * a unit - * @return the next unit in the database, or null if there is no such unit. - */ - public VocalizationUnit getNextUnit(VocalizationUnit u) { - if (u == null || u.index >= backchannelUnits.length - 1 || u.index < 0) - return null; - return backchannelUnits[u.index + 1]; - } - - /** - * Return the unit preceding the given unit in the original database. - * - * @param u - * a unit - * @return the previous unit in the database, or null if there is no such unit. - */ - public VocalizationUnit getPreviousUnit(VocalizationUnit u) { - if (u == null || u.index >= backchannelUnits.length || u.index <= 0) - return null; - return backchannelUnits[u.index - 1]; - } - - /** - * Determine whether the unit number i is an "edge" unit, i.e. a unit marking the start or the end of an utterance. - * - * @param i - * The index of the considered unit. - * @return true if the unit is an edge unit in the unit file, false otherwise - */ - public boolean isEdgeUnit(int i) { - return backchannelUnits[i].isEdgeUnit(); - } - - public static void main(String[] args) throws Exception { - String fileName = "/home/sathish/Work/dfki399/backchannel/mary_files/BCCphoneUnits.mry"; - VocalizationUnitFileReader bcUfr = new VocalizationUnitFileReader(); - bcUfr.load(fileName); - } -} From 267c126d220179e007fcff68f66240d3ee405d74 Mon Sep 17 00:00:00 2001 From: Tristan Hamilton Date: Mon, 2 Feb 2015 15:17:56 +0100 Subject: [PATCH 04/31] clean runtime of all references to vocalizations --- .../main/java/marytts/modules/Synthesis.java | 13 ------------ .../java/marytts/modules/synthesis/Voice.java | 10 ---------- .../server/http/BaseHttpRequestHandler.java | 1 - .../server/http/InfoRequestHandler.java | 14 ------------- .../java/marytts/util/MaryRuntimeUtils.java | 20 ------------------- 5 files changed, 58 deletions(-) diff --git a/marytts-runtime/src/main/java/marytts/modules/Synthesis.java b/marytts-runtime/src/main/java/marytts/modules/Synthesis.java index e291395496..7168ef07b8 100644 --- a/marytts-runtime/src/main/java/marytts/modules/Synthesis.java +++ b/marytts-runtime/src/main/java/marytts/modules/Synthesis.java @@ -144,19 +144,6 @@ public MaryData process(MaryData d) throws Exception { Element v = (Element) MaryDomUtils.getAncestor(element, MaryXML.VOICE); Element s = (Element) MaryDomUtils.getAncestor(element, MaryXML.SENTENCE); - // Check for non-verbal elements - if (element.getNodeName().equals(MaryXML.NONVERBAL)) { - if (v != null) { - Voice newvoice = Voice.getVoice(v); - if (newvoice != null && newvoice.hasVocalizationSupport()) { - AudioInputStream ais = newvoice.getVocalizationSynthesizer().synthesize(newvoice, d.getAudioFileFormat(), - element); - result.appendAudio(ais); - } - } - continue; - } - // Chunk at boundaries between voice sections if (v == null) { if (currentVoiceElement != null) { diff --git a/marytts-runtime/src/main/java/marytts/modules/synthesis/Voice.java b/marytts-runtime/src/main/java/marytts/modules/synthesis/Voice.java index 900b586930..41154a68d9 100644 --- a/marytts-runtime/src/main/java/marytts/modules/synthesis/Voice.java +++ b/marytts-runtime/src/main/java/marytts/modules/synthesis/Voice.java @@ -74,7 +74,6 @@ import marytts.unitselection.interpolation.InterpolatingVoice; import marytts.util.MaryRuntimeUtils; import marytts.util.MaryUtils; -import marytts.vocalizations.VocalizationSynthesizer; import org.apache.log4j.Logger; import org.w3c.dom.Element; @@ -154,7 +153,6 @@ public int compare(Voice v1, Voice v2) { private Vector preferredModules; private Lexicon lexicon; private boolean vocalizationSupport; - private VocalizationSynthesizer vocalizationSynthesizer; protected DirectedGraph durationGraph; protected DirectedGraph f0Graph; protected FeatureFileReader f0ContourFeatures; @@ -229,10 +227,6 @@ private void init() throws MaryConfigurationException, NoSuchPropertyException, String lexiconClass = MaryProperties.getProperty(header + ".lexiconClass"); String lexiconName = MaryProperties.getProperty(header + ".lexicon"); lexicon = getLexicon(lexiconClass, lexiconName); - vocalizationSupport = MaryProperties.getBoolean(header + ".vocalizationSupport", false); - if (vocalizationSupport) { - vocalizationSynthesizer = new VocalizationSynthesizer(this); - } loadOldStyleProsodyModels(header); loadAcousticModels(header); @@ -513,10 +507,6 @@ public boolean hasVocalizationSupport() { return vocalizationSupport; } - public VocalizationSynthesizer getVocalizationSynthesizer() { - return vocalizationSynthesizer; - } - /** * Get any styles supported by this voice. * diff --git a/marytts-runtime/src/main/java/marytts/server/http/BaseHttpRequestHandler.java b/marytts-runtime/src/main/java/marytts/server/http/BaseHttpRequestHandler.java index ffc5b7505d..4e66252d4c 100644 --- a/marytts-runtime/src/main/java/marytts/server/http/BaseHttpRequestHandler.java +++ b/marytts-runtime/src/main/java/marytts/server/http/BaseHttpRequestHandler.java @@ -49,7 +49,6 @@ import marytts.util.MaryUtils; import marytts.util.http.Address; import marytts.util.string.StringUtils; -import marytts.vocalizations.VocalizationSynthesizer; import org.apache.http.Header; import org.apache.http.HttpEntityEnclosingRequest; diff --git a/marytts-runtime/src/main/java/marytts/server/http/InfoRequestHandler.java b/marytts-runtime/src/main/java/marytts/server/http/InfoRequestHandler.java index 765d7f717b..1630e8b09a 100644 --- a/marytts-runtime/src/main/java/marytts/server/http/InfoRequestHandler.java +++ b/marytts-runtime/src/main/java/marytts/server/http/InfoRequestHandler.java @@ -186,20 +186,6 @@ public String callCorrectMethod(Map queryItems, HttpResponse res } }); - methodsMapping.put("vocalizations", new CorrectMethod() { - @Override - public String callCorrectMethod(Map queryItems, HttpResponse response) { - if (queryItems != null) { - String voice = queryItems.get("voice"); - if (voice != null) { - return MaryRuntimeUtils.getVocalizations(voice); - } - } - MaryHttpServerUtils.errorMissingQueryParameter(response, "'voice'"); - return null; - } - }); - methodsMapping.put("styles", new CorrectMethod() { @Override public String callCorrectMethod(Map queryItems, HttpResponse response) { diff --git a/marytts-runtime/src/main/java/marytts/util/MaryRuntimeUtils.java b/marytts-runtime/src/main/java/marytts/util/MaryRuntimeUtils.java index 644f5c0a7a..5016dc6fe4 100644 --- a/marytts-runtime/src/main/java/marytts/util/MaryRuntimeUtils.java +++ b/marytts-runtime/src/main/java/marytts/util/MaryRuntimeUtils.java @@ -60,7 +60,6 @@ import marytts.util.data.audio.MaryAudioUtils; import marytts.util.dom.MaryDomUtils; import marytts.util.string.StringUtils; -import marytts.vocalizations.VocalizationSynthesizer; import org.w3c.dom.Element; @@ -479,25 +478,6 @@ public static String getVoiceExampleText(String voiceName) { return ""; } - /** - * For the voice with the given name, return the list of vocalizations supported by this voice, one vocalization per line. - * These values can be used in the "name" attribute of the vocalization tag. - * - * @param voiceName - * @return the list of vocalizations, or the empty string if the voice does not support vocalizations. - */ - public static String getVocalizations(String voiceName) { - Voice v = Voice.getVoice(voiceName); - if (v == null || !v.hasVocalizationSupport()) { - return ""; - } - VocalizationSynthesizer vs = v.getVocalizationSynthesizer(); - assert vs != null; - String[] vocalizations = vs.listAvailableVocalizations(); - assert vocalizations != null; - return StringUtils.toString(vocalizations); - } - /** * For the voice with the given name, return the list of styles supported by this voice, if any, one style per line. These * values can be used as the global "style" value in a synthesis request, or in the "style" attribute of the MaryXML prosody From 3d1093d3484e4eef8c01b6ebe70603c41dbc11f9 Mon Sep 17 00:00:00 2001 From: Tristan Hamilton Date: Thu, 5 Feb 2015 13:49:55 +0100 Subject: [PATCH 05/31] remove unit-selection package from runtime -> Compile Errors --- .../UnitSelectionSynthesizer.java | 270 ---- .../unitselection/UnitSelectionVoice.java | 325 ---- .../analysis/HnmVoiceDataDumper.java | 126 -- .../marytts/unitselection/analysis/Phone.java | 748 ---------- .../analysis/ProsodyAnalyzer.java | 437 ------ .../analysis/VoiceDataDumper.java | 351 ----- .../concat/BaseUnitConcatenator.java | 315 ---- .../DatagramOverlapDoubleDataSource.java | 148 -- .../concat/FdpsolaUnitConcatenator.java | 601 -------- .../concat/HnmUnitConcatenator.java | 251 ---- .../concat/OverlapUnitConcatenator.java | 206 --- .../concat/UnitConcatenator.java | 61 - .../unitselection/data/DiphoneUnit.java | 65 - .../data/DiphoneUnitDatabase.java | 159 -- .../unitselection/data/FeatureFileReader.java | 218 --- .../data/FloatArrayDatagram.java | 67 - .../data/HalfPhoneFeatureFileReader.java | 99 -- .../unitselection/data/HnmDatagram.java | 149 -- .../unitselection/data/HnmTimelineReader.java | 248 ---- .../unitselection/data/LPCDatagram.java | 229 --- .../unitselection/data/LPCTimelineReader.java | 100 -- .../unitselection/data/MCepDatagram.java | 175 --- .../data/MCepTimelineReader.java | 85 -- .../unitselection/data/SCostFileReader.java | 140 -- .../marytts/unitselection/data/Sentence.java | 63 - .../unitselection/data/SentenceIterator.java | 152 -- .../marytts/unitselection/data/Syllable.java | 77 - .../unitselection/data/SyllableIterator.java | 151 -- .../unitselection/data/TimelineReader.java | 1304 ----------------- .../java/marytts/unitselection/data/Unit.java | 85 -- .../unitselection/data/UnitDatabase.java | 195 --- .../unitselection/data/UnitFileReader.java | 200 --- .../InterpolatingSynthesizer.java | 203 --- .../interpolation/InterpolatingVoice.java | 119 -- .../select/DiphoneFFRTargetCostFunction.java | 142 -- .../unitselection/select/DiphoneTarget.java | 71 - .../select/DiphoneUnitSelector.java | 69 - .../select/FFRTargetCostFunction.java | 386 ----- .../HalfPhoneFFRTargetCostFunction.java | 243 --- .../unitselection/select/HalfPhoneTarget.java | 60 - .../select/HalfPhoneUnitSelector.java | 57 - .../select/JoinCostFeatures.java | 623 -------- .../select/JoinCostFunction.java | 77 - .../unitselection/select/JoinModelCost.java | 217 --- .../select/PrecompiledJoinCostReader.java | 156 -- .../unitselection/select/SelectedUnit.java | 76 - .../select/StatisticalCostFunction.java | 44 - .../select/StatisticalModelCost.java | 143 -- .../marytts/unitselection/select/Target.java | 191 --- .../select/TargetCostFunction.java | 126 -- .../unitselection/select/UnitSelector.java | 158 -- .../unitselection/select/viterbi/Viterbi.java | 532 ------- .../select/viterbi/ViterbiCandidate.java | 142 -- .../select/viterbi/ViterbiPath.java | 110 -- .../select/viterbi/ViterbiPoint.java | 129 -- .../weightingfunctions/WeightFunc.java | 49 - .../WeightFunctionManager.java | 99 -- .../weightingfunctions/WeightingFunction.java | 119 -- 58 files changed, 12141 deletions(-) delete mode 100644 marytts-runtime/src/main/java/marytts/unitselection/UnitSelectionSynthesizer.java delete mode 100644 marytts-runtime/src/main/java/marytts/unitselection/UnitSelectionVoice.java delete mode 100644 marytts-runtime/src/main/java/marytts/unitselection/analysis/HnmVoiceDataDumper.java delete mode 100644 marytts-runtime/src/main/java/marytts/unitselection/analysis/Phone.java delete mode 100644 marytts-runtime/src/main/java/marytts/unitselection/analysis/ProsodyAnalyzer.java delete mode 100644 marytts-runtime/src/main/java/marytts/unitselection/analysis/VoiceDataDumper.java delete mode 100644 marytts-runtime/src/main/java/marytts/unitselection/concat/BaseUnitConcatenator.java delete mode 100644 marytts-runtime/src/main/java/marytts/unitselection/concat/DatagramOverlapDoubleDataSource.java delete mode 100644 marytts-runtime/src/main/java/marytts/unitselection/concat/FdpsolaUnitConcatenator.java delete mode 100644 marytts-runtime/src/main/java/marytts/unitselection/concat/HnmUnitConcatenator.java delete mode 100644 marytts-runtime/src/main/java/marytts/unitselection/concat/OverlapUnitConcatenator.java delete mode 100644 marytts-runtime/src/main/java/marytts/unitselection/concat/UnitConcatenator.java delete mode 100644 marytts-runtime/src/main/java/marytts/unitselection/data/DiphoneUnit.java delete mode 100644 marytts-runtime/src/main/java/marytts/unitselection/data/DiphoneUnitDatabase.java delete mode 100644 marytts-runtime/src/main/java/marytts/unitselection/data/FeatureFileReader.java delete mode 100644 marytts-runtime/src/main/java/marytts/unitselection/data/FloatArrayDatagram.java delete mode 100644 marytts-runtime/src/main/java/marytts/unitselection/data/HalfPhoneFeatureFileReader.java delete mode 100644 marytts-runtime/src/main/java/marytts/unitselection/data/HnmDatagram.java delete mode 100644 marytts-runtime/src/main/java/marytts/unitselection/data/HnmTimelineReader.java delete mode 100644 marytts-runtime/src/main/java/marytts/unitselection/data/LPCDatagram.java delete mode 100644 marytts-runtime/src/main/java/marytts/unitselection/data/LPCTimelineReader.java delete mode 100644 marytts-runtime/src/main/java/marytts/unitselection/data/MCepDatagram.java delete mode 100644 marytts-runtime/src/main/java/marytts/unitselection/data/MCepTimelineReader.java delete mode 100644 marytts-runtime/src/main/java/marytts/unitselection/data/SCostFileReader.java delete mode 100644 marytts-runtime/src/main/java/marytts/unitselection/data/Sentence.java delete mode 100644 marytts-runtime/src/main/java/marytts/unitselection/data/SentenceIterator.java delete mode 100644 marytts-runtime/src/main/java/marytts/unitselection/data/Syllable.java delete mode 100644 marytts-runtime/src/main/java/marytts/unitselection/data/SyllableIterator.java delete mode 100644 marytts-runtime/src/main/java/marytts/unitselection/data/TimelineReader.java delete mode 100644 marytts-runtime/src/main/java/marytts/unitselection/data/Unit.java delete mode 100644 marytts-runtime/src/main/java/marytts/unitselection/data/UnitDatabase.java delete mode 100644 marytts-runtime/src/main/java/marytts/unitselection/data/UnitFileReader.java delete mode 100644 marytts-runtime/src/main/java/marytts/unitselection/interpolation/InterpolatingSynthesizer.java delete mode 100644 marytts-runtime/src/main/java/marytts/unitselection/interpolation/InterpolatingVoice.java delete mode 100644 marytts-runtime/src/main/java/marytts/unitselection/select/DiphoneFFRTargetCostFunction.java delete mode 100644 marytts-runtime/src/main/java/marytts/unitselection/select/DiphoneTarget.java delete mode 100644 marytts-runtime/src/main/java/marytts/unitselection/select/DiphoneUnitSelector.java delete mode 100644 marytts-runtime/src/main/java/marytts/unitselection/select/FFRTargetCostFunction.java delete mode 100644 marytts-runtime/src/main/java/marytts/unitselection/select/HalfPhoneFFRTargetCostFunction.java delete mode 100644 marytts-runtime/src/main/java/marytts/unitselection/select/HalfPhoneTarget.java delete mode 100644 marytts-runtime/src/main/java/marytts/unitselection/select/HalfPhoneUnitSelector.java delete mode 100644 marytts-runtime/src/main/java/marytts/unitselection/select/JoinCostFeatures.java delete mode 100644 marytts-runtime/src/main/java/marytts/unitselection/select/JoinCostFunction.java delete mode 100644 marytts-runtime/src/main/java/marytts/unitselection/select/JoinModelCost.java delete mode 100644 marytts-runtime/src/main/java/marytts/unitselection/select/PrecompiledJoinCostReader.java delete mode 100644 marytts-runtime/src/main/java/marytts/unitselection/select/SelectedUnit.java delete mode 100644 marytts-runtime/src/main/java/marytts/unitselection/select/StatisticalCostFunction.java delete mode 100644 marytts-runtime/src/main/java/marytts/unitselection/select/StatisticalModelCost.java delete mode 100644 marytts-runtime/src/main/java/marytts/unitselection/select/Target.java delete mode 100644 marytts-runtime/src/main/java/marytts/unitselection/select/TargetCostFunction.java delete mode 100644 marytts-runtime/src/main/java/marytts/unitselection/select/UnitSelector.java delete mode 100644 marytts-runtime/src/main/java/marytts/unitselection/select/viterbi/Viterbi.java delete mode 100644 marytts-runtime/src/main/java/marytts/unitselection/select/viterbi/ViterbiCandidate.java delete mode 100644 marytts-runtime/src/main/java/marytts/unitselection/select/viterbi/ViterbiPath.java delete mode 100644 marytts-runtime/src/main/java/marytts/unitselection/select/viterbi/ViterbiPoint.java delete mode 100644 marytts-runtime/src/main/java/marytts/unitselection/weightingfunctions/WeightFunc.java delete mode 100644 marytts-runtime/src/main/java/marytts/unitselection/weightingfunctions/WeightFunctionManager.java delete mode 100644 marytts-runtime/src/main/java/marytts/unitselection/weightingfunctions/WeightingFunction.java diff --git a/marytts-runtime/src/main/java/marytts/unitselection/UnitSelectionSynthesizer.java b/marytts-runtime/src/main/java/marytts/unitselection/UnitSelectionSynthesizer.java deleted file mode 100644 index 950bc0322b..0000000000 --- a/marytts-runtime/src/main/java/marytts/unitselection/UnitSelectionSynthesizer.java +++ /dev/null @@ -1,270 +0,0 @@ -/** - * Copyright 2006 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.unitselection; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.PrintWriter; -import java.io.StringReader; -import java.io.StringWriter; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Iterator; -import java.util.List; -import java.util.Locale; -import java.util.StringTokenizer; - -import javax.sound.sampled.AudioFormat; -import javax.sound.sampled.AudioInputStream; - -import marytts.datatypes.MaryData; -import marytts.datatypes.MaryDataType; -import marytts.datatypes.MaryXML; -import marytts.exceptions.SynthesisException; -import marytts.modules.synthesis.Voice; -import marytts.modules.synthesis.WaveformSynthesizer; -import marytts.modules.synthesis.Voice.Gender; -import marytts.server.MaryProperties; -import marytts.unitselection.concat.UnitConcatenator; -import marytts.unitselection.concat.BaseUnitConcatenator.UnitData; -import marytts.unitselection.data.Unit; -import marytts.unitselection.data.UnitDatabase; -import marytts.unitselection.select.HalfPhoneTarget; -import marytts.unitselection.select.SelectedUnit; -import marytts.unitselection.select.Target; -import marytts.unitselection.select.UnitSelector; -import marytts.util.MaryUtils; -import marytts.util.dom.MaryNormalisedWriter; -import marytts.util.dom.NameNodeFilter; - -import org.apache.log4j.Level; -import org.apache.log4j.Logger; -import org.w3c.dom.Element; -import org.w3c.dom.traversal.DocumentTraversal; -import org.w3c.dom.traversal.NodeFilter; -import org.w3c.dom.traversal.TreeWalker; - -/** - * Builds and synthesizes unit selection voices - * - * @author Marc Schröder, Anna Hunecke - * - */ - -public class UnitSelectionSynthesizer implements WaveformSynthesizer { - /** - * A map with Voice objects as keys, and Lists of UtteranceProcessors as values. Idea: For a given voice, find the list of - * utterance processors to apply. - */ - private Logger logger; - - public UnitSelectionSynthesizer() { - } - - /** - * Start up the waveform synthesizer. This must be called once before calling synthesize(). - */ - public void startup() throws Exception { - logger = MaryUtils.getLogger("UnitSelectionSynthesizer"); - // Register UnitSelection voices: - logger.debug("Register UnitSelection voices:"); - List voiceNames = MaryProperties.getList("unitselection.voices.list"); - for (String voiceName : voiceNames) { - long time = System.currentTimeMillis(); - Voice unitSelVoice = new UnitSelectionVoice(voiceName, this); - logger.debug("Voice '" + unitSelVoice + "'"); - Voice.registerVoice(unitSelVoice); - long newtime = System.currentTimeMillis() - time; - logger.info("Loading of voice " + voiceName + " took " + newtime + " milliseconds"); - } - logger.info("started."); - } - - /** - * Perform a power-on self test by processing some example input data. - * - * @throws Error - * if the module does not work properly. - */ - public void powerOnSelfTest() throws Error { - try { - Collection myVoices = Voice.getAvailableVoices(this); - if (myVoices.size() == 0) { - return; - } - UnitSelectionVoice unitSelVoice = (UnitSelectionVoice) myVoices.iterator().next(); - assert unitSelVoice != null; - MaryData in = new MaryData(MaryDataType.get("ACOUSTPARAMS"), unitSelVoice.getLocale()); - if (!unitSelVoice.getDomain().equals("general")) { - logger.info("Cannot perform power-on self test using limited-domain voice '" + unitSelVoice.getName() - + "' - skipping."); - return; - } - String exampleText = MaryDataType.ACOUSTPARAMS.exampleText(unitSelVoice.getLocale()); - if (exampleText != null) { - in.readFrom(new StringReader(exampleText)); - in.setDefaultVoice(unitSelVoice); - if (in == null) { - System.out.println(exampleText + " is null"); - } - List tokensAndBoundaries = new ArrayList(); - TreeWalker tw = ((DocumentTraversal) in.getDocument()).createTreeWalker(in.getDocument(), - NodeFilter.SHOW_ELEMENT, new NameNodeFilter(new String[] { MaryXML.TOKEN, MaryXML.BOUNDARY }), false); - Element el = null; - while ((el = (Element) tw.nextNode()) != null) - tokensAndBoundaries.add(el); - AudioInputStream ais = synthesize(tokensAndBoundaries, unitSelVoice, null); - assert ais != null; - } else { - logger.debug("No example text -- no power-on self test!"); - } - } catch (Throwable t) { - t.printStackTrace(); - throw new Error("Module " + toString() + ": Power-on self test failed.", t); - } - logger.info("Power-on self test complete."); - } - - /** - * {@inheritDoc} - */ - public AudioInputStream synthesize(List tokensAndBoundaries, Voice voice, String outputParams) - throws SynthesisException { - assert voice instanceof UnitSelectionVoice; - UnitSelectionVoice v = (UnitSelectionVoice) voice; - UnitDatabase udb = v.getDatabase(); - // Select: - UnitSelector unitSel = v.getUnitSelector(); - UnitConcatenator unitConcatenator; - if (outputParams != null && outputParams.contains("MODIFICATION")) { - unitConcatenator = v.getModificationConcatenator(); - } else { - unitConcatenator = v.getConcatenator(); - } - // TODO: check if we actually need to access v.getDatabase() here - UnitDatabase database = v.getDatabase(); - logger.debug("Selecting units with a " + unitSel.getClass().getName() + " from a " + database.getClass().getName()); - List selectedUnits = unitSel.selectUnits(tokensAndBoundaries, voice); - // if (logger.getEffectiveLevel().equals(Level.DEBUG)) { - // StringWriter sw = new StringWriter(); - // PrintWriter pw = new PrintWriter(sw); - // for (Iterator selIt=selectedUnits.iterator(); selIt.hasNext(); ) - // pw.println(selIt.next()); - // logger.debug("Units selected:\n"+sw.toString()); - // } - - // Concatenate: - logger.debug("Now creating audio with a " + unitConcatenator.getClass().getName()); - AudioInputStream audio = null; - try { - audio = unitConcatenator.getAudio(selectedUnits); - } catch (IOException ioe) { - StringWriter sw = new StringWriter(); - PrintWriter pw = new PrintWriter(sw); - for (Iterator selIt = selectedUnits.iterator(); selIt.hasNext();) - pw.println(selIt.next()); - throw new SynthesisException("Problems generating audio for unit chain: " + sw.toString(), ioe); - } - - // Propagate unit durations to XML tree: - float endInSeconds = 0; - float durLeftHalfInSeconds = 0; - String unitString = ""; - String unitAttrName = "units"; // name of the attribute that is added for unit selection diagnostics - for (SelectedUnit su : selectedUnits) { - Target t = su.getTarget(); - boolean halfphone = (t instanceof HalfPhoneTarget); - Object concatenationData = su.getConcatenationData(); - assert concatenationData instanceof UnitData; - UnitData unitData = (UnitData) concatenationData; - Unit unit = su.getUnit(); - - // For the unit durations, keep record in floats because of precision; - // convert to millis only at export time, and re-compute duration in millis - // from the end in millis, to avoid discrepancies due to rounding - int unitDurationInSamples = unitData.getUnitDuration(); - float unitDurationInSeconds = unitDurationInSamples / (float) database.getUnitFileReader().getSampleRate(); - int prevEndInMillis = (int) (1000 * endInSeconds); - endInSeconds += unitDurationInSeconds; - int endInMillis = (int) (1000 * endInSeconds); - int unitDurationInMillis = endInMillis - prevEndInMillis; - unitString = t.getName() + " " + udb.getFilename(unit) + " " + unit.index + " " + unitDurationInSeconds; - if (halfphone) { - if (((HalfPhoneTarget) t).isLeftHalf()) { - durLeftHalfInSeconds = unitDurationInSeconds; - } else { // right half - // re-compute unit duration from both halves - float totalUnitDurInSeconds = durLeftHalfInSeconds + unitDurationInSeconds; - float prevEndInSeconds = endInSeconds - totalUnitDurInSeconds; - prevEndInMillis = (int) (1000 * prevEndInSeconds); - unitDurationInMillis = endInMillis - prevEndInMillis; - durLeftHalfInSeconds = 0; - } - } - - Element maryxmlElement = t.getMaryxmlElement(); - if (maryxmlElement != null) { - if (maryxmlElement.getNodeName().equals(MaryXML.PHONE)) { - if (!maryxmlElement.hasAttribute("d") || !maryxmlElement.hasAttribute("end")) { - throw new IllegalStateException("No duration information in MaryXML -- check log file" - + " for messages warning about unloadable acoustic models" - + " instead of voice-specific acoustic feature predictors"); - } - // int oldD = Integer.parseInt(maryxmlElement.getAttribute("d")); - // int oldEnd = Integer.parseInt(maryxmlElement.getAttribute("end")); - // double doubleEnd = Double.parseDouble(maryxmlElement.getAttribute("end")); - // int oldEnd = (int)(doubleEnd * 1000); - maryxmlElement.setAttribute("d", String.valueOf(unitDurationInMillis)); - maryxmlElement.setAttribute("end", String.valueOf(endInSeconds)); - // the following messes up all end values! - // if (oldEnd == oldD) { - // // start new end computation - // endInSeconds = unitDurationInSeconds; - // } - } else { // not a PHONE - assert maryxmlElement.getNodeName().equals(MaryXML.BOUNDARY); - maryxmlElement.setAttribute("duration", String.valueOf(unitDurationInMillis)); - } - if (maryxmlElement.hasAttribute(unitAttrName)) { - String prevUnitString = maryxmlElement.getAttribute(unitAttrName); - maryxmlElement.setAttribute(unitAttrName, prevUnitString + "; " + unitString); - } else { - maryxmlElement.setAttribute(unitAttrName, unitString); - } - } else { - logger.debug("Unit " + su.getTarget().getName() + " of length " + unitDurationInMillis - + " ms has no maryxml element."); - } - } - if (logger.getEffectiveLevel().equals(Level.DEBUG)) { - try { - MaryNormalisedWriter writer = new MaryNormalisedWriter(); - ByteArrayOutputStream debugOut = new ByteArrayOutputStream(); - writer.output(tokensAndBoundaries.get(0).getOwnerDocument(), debugOut); - logger.debug("Propagating the realised unit durations to the XML tree: \n" + debugOut.toString()); - } catch (Exception e) { - logger.warn("Problem writing XML to logfile: " + e); - } - } - - return audio; - } - -} diff --git a/marytts-runtime/src/main/java/marytts/unitselection/UnitSelectionVoice.java b/marytts-runtime/src/main/java/marytts/unitselection/UnitSelectionVoice.java deleted file mode 100644 index c11d484424..0000000000 --- a/marytts-runtime/src/main/java/marytts/unitselection/UnitSelectionVoice.java +++ /dev/null @@ -1,325 +0,0 @@ -/** - * Portions Copyright 2006 DFKI GmbH. - * Portions Copyright 2001 Sun Microsystems, Inc. - * Portions Copyright 1999-2001 Language Technologies Institute, - * Carnegie Mellon University. - * All Rights Reserved. Use is subject to license terms. - * - * Permission is hereby granted, free of charge, to use and distribute - * this software and its documentation without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of this work, and to - * permit persons to whom this work is furnished to do so, subject to - * the following conditions: - * - * 1. The code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * 2. Any modifications must be clearly marked as such. - * 3. Original authors' names are not deleted. - * 4. The authors' names are not used to endorse or promote products - * derived from this software without specific prior written - * permission. - * - * DFKI GMBH AND THE CONTRIBUTORS TO THIS WORK DISCLAIM ALL WARRANTIES WITH - * REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL DFKI GMBH NOR THE - * CONTRIBUTORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL - * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR - * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS - * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF - * THIS SOFTWARE. - */ -package marytts.unitselection; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.lang.reflect.Constructor; -import java.util.Locale; - -import javax.sound.sampled.AudioFormat; - -import marytts.cart.CART; -import marytts.cart.io.MaryCARTReader; -import marytts.exceptions.MaryConfigurationException; -import marytts.features.FeatureDefinition; -import marytts.features.FeatureProcessorManager; -import marytts.features.FeatureRegistry; -import marytts.modules.synthesis.Voice; -import marytts.modules.synthesis.WaveformSynthesizer; -import marytts.server.MaryProperties; -import marytts.unitselection.concat.FdpsolaUnitConcatenator; -import marytts.unitselection.concat.UnitConcatenator; -import marytts.unitselection.data.TimelineReader; -import marytts.unitselection.data.UnitDatabase; -import marytts.unitselection.data.UnitFileReader; -import marytts.unitselection.select.JoinCostFunction; -import marytts.unitselection.select.JoinModelCost; -import marytts.unitselection.select.StatisticalCostFunction; -import marytts.unitselection.select.TargetCostFunction; -import marytts.unitselection.select.UnitSelector; - -/** - * A Unit Selection Voice - * - */ -public class UnitSelectionVoice extends Voice { - - protected UnitDatabase database; - protected UnitSelector unitSelector; - protected UnitConcatenator concatenator; - protected UnitConcatenator modificationConcatenator; - protected String domain; - protected String name; - protected CART[] f0Carts; - protected String exampleText; - - public UnitSelectionVoice(String name, WaveformSynthesizer synthesizer) throws MaryConfigurationException { - super(name, synthesizer); - - try { - this.name = name; - String header = "voice." + name; - - domain = MaryProperties.needProperty(header + ".domain"); - InputStream exampleTextStream = null; - if (!domain.equals("general")) { // limited domain voices must have example text; - exampleTextStream = MaryProperties.needStream(header + ".exampleTextFile"); - } else { // general domain voices can have example text: - exampleTextStream = MaryProperties.getStream(header + ".exampleTextFile"); - } - if (exampleTextStream != null) { - readExampleText(exampleTextStream); - } - - FeatureProcessorManager featProcManager = FeatureRegistry.getFeatureProcessorManager(this); - if (featProcManager == null) - featProcManager = FeatureRegistry.getFeatureProcessorManager(getLocale()); - if (featProcManager == null) - throw new MaryConfigurationException("No feature processor manager for voice '" + name + "' (locale " - + getLocale() + ")"); - - // build and load targetCostFunction - logger.debug("...loading target cost function..."); - String featureFileName = MaryProperties.needFilename(header + ".featureFile"); - InputStream targetWeightStream = MaryProperties.getStream(header + ".targetCostWeights"); - String targetCostClass = MaryProperties.needProperty(header + ".targetCostClass"); - TargetCostFunction targetFunction = (TargetCostFunction) Class.forName(targetCostClass).newInstance(); - targetFunction.load(featureFileName, targetWeightStream, featProcManager); - - // build joinCostFunction - logger.debug("...loading join cost function..."); - String joinCostClass = MaryProperties.needProperty(header + ".joinCostClass"); - JoinCostFunction joinFunction = (JoinCostFunction) Class.forName(joinCostClass).newInstance(); - if (joinFunction instanceof JoinModelCost) { - ((JoinModelCost) joinFunction).setFeatureDefinition(targetFunction.getFeatureDefinition()); - } - joinFunction.init(header); - - // build sCost function - StatisticalCostFunction sCostFunction = null; - boolean useSCost = MaryProperties.getBoolean(header + ".useSCost", false); - if (useSCost) { - logger.debug("...loading scost function..."); - String sCostClass = MaryProperties.needProperty(header + ".sCostClass"); - sCostFunction = (StatisticalCostFunction) Class.forName(sCostClass).newInstance(); - sCostFunction.init(header); - } - - // Build the various file readers - logger.debug("...loading units file..."); - String unitReaderClass = MaryProperties.needProperty(header + ".unitReaderClass"); - String unitsFile = MaryProperties.needFilename(header + ".unitsFile"); - UnitFileReader unitReader = (UnitFileReader) Class.forName(unitReaderClass).newInstance(); - unitReader.load(unitsFile); - - logger.debug("...loading cart file..."); - // String cartReaderClass = MaryProperties.needProperty(header+".cartReaderClass"); - InputStream cartStream = MaryProperties.needStream(header + ".cartFile"); - CART cart = new MaryCARTReader().loadFromStream(cartStream); - cartStream.close(); - // get the backtrace information - int backtrace = MaryProperties.getInteger(header + ".cart.backtrace", 100); - - logger.debug("...loading audio time line..."); - String timelineReaderClass = MaryProperties.needProperty(header + ".audioTimelineReaderClass"); - String timelineFile = MaryProperties.needFilename(header + ".audioTimelineFile"); - Class theClass = Class.forName(timelineReaderClass).asSubclass(TimelineReader.class); - // Now invoke Constructor with one String argument - Class[] constructorArgTypes = new Class[] { String.class }; - Object[] args = new Object[] { timelineFile }; - Constructor constructor = (Constructor) theClass - .getConstructor(constructorArgTypes); - TimelineReader timelineReader = constructor.newInstance(args); - - // optionally, get basename timeline - String basenameTimelineFile = MaryProperties.getFilename(header + ".basenameTimeline"); - TimelineReader basenameTimelineReader = null; - if (basenameTimelineFile != null) { - logger.debug("...loading basename time line..."); - basenameTimelineReader = new TimelineReader(basenameTimelineFile); - } - - // build and load database - logger.debug("...instantiating database..."); - String databaseClass = MaryProperties.needProperty(header + ".databaseClass"); - database = (UnitDatabase) Class.forName(databaseClass).newInstance(); - if (useSCost) { - database.load(targetFunction, joinFunction, sCostFunction, unitReader, cart, timelineReader, - basenameTimelineReader, backtrace); - } else { - database.load(targetFunction, joinFunction, unitReader, cart, timelineReader, basenameTimelineReader, backtrace); - } - - // build Selector - logger.debug("...instantiating unit selector..."); - String selectorClass = MaryProperties.needProperty(header + ".selectorClass"); - unitSelector = (UnitSelector) Class.forName(selectorClass).newInstance(); - float targetCostWeights = Float.parseFloat(MaryProperties.getProperty(header + ".viterbi.wTargetCosts", "0.33")); - int beamSize = MaryProperties.getInteger(header + ".viterbi.beamsize", 100); - if (!useSCost) { - unitSelector.load(database, targetCostWeights, beamSize); - } else { - float sCostWeights = Float.parseFloat(MaryProperties.getProperty(header + ".viterbi.wSCosts", "0.33")); - unitSelector.load(database, targetCostWeights, sCostWeights, beamSize); - } - - // samplingRate -> bin, audioformat -> concatenator - // build Concatenator - logger.debug("...instantiating unit concatenator..."); - String concatenatorClass = MaryProperties.needProperty(header + ".concatenatorClass"); - concatenator = (UnitConcatenator) Class.forName(concatenatorClass).newInstance(); - concatenator.load(database); - - // TODO: this can be deleted at the same time as CARTF0Modeller - // see if there are any voice-specific duration and f0 models to load - f0Carts = null; - InputStream leftF0CartStream = MaryProperties.getStream(header + ".f0.cart.left"); - if (leftF0CartStream != null) { - logger.debug("...loading f0 trees..."); - f0Carts = new CART[3]; - f0Carts[0] = new MaryCARTReader().loadFromStream(leftF0CartStream); - leftF0CartStream.close(); - // mid cart: - InputStream midF0CartStream = MaryProperties.needStream(header + ".f0.cart.mid"); - f0Carts[1] = new MaryCARTReader().loadFromStream(midF0CartStream); - midF0CartStream.close(); - // right cart: - InputStream rightF0CartStream = MaryProperties.needStream(header + ".f0.cart.right"); - f0Carts[2] = new MaryCARTReader().loadFromStream(rightF0CartStream); - rightF0CartStream.close(); - } - } catch (MaryConfigurationException mce) { - throw mce; - } catch (Exception ex) { - throw new MaryConfigurationException("Cannot build unit selection voice '" + name + "'", ex); - } - - } - - /** - * Gets the database of this voice - * - * @return the database - */ - public UnitDatabase getDatabase() { - return database; - } - - /** - * Gets the unit selector of this voice - * - * @return the unit selector - */ - public UnitSelector getUnitSelector() { - return unitSelector; - } - - /** - * Gets the unit concatenator of this voice - * - * @return the unit selector - */ - public UnitConcatenator getConcatenator() { - return concatenator; - } - - /** - * Get the modification UnitConcatenator of this voice - * - * @return the modifying UnitConcatenator - */ - public UnitConcatenator getModificationConcatenator() { - if (modificationConcatenator == null) { - // get sensible minimum and maximum values: - try { - // initialize with values from properties: - double minTimeScaleFactor = Double.parseDouble(MaryProperties.getProperty("voice." + name - + ".prosody.modification.duration.factor.minimum")); - double maxTimeScaleFactor = Double.parseDouble(MaryProperties.getProperty("voice." + name - + ".prosody.modification.duration.factor.maximum")); - double minPitchScaleFactor = Double.parseDouble(MaryProperties.getProperty("voice." + name - + ".prosody.modification.f0.factor.minimum")); - double maxPitchScaleFactor = Double.parseDouble(MaryProperties.getProperty("voice." + name - + ".prosody.modification.f0.factor.maximum")); - logger.debug("Initializing FD-PSOLA unit concatenator with the following parameter thresholds:"); - logger.debug("minimum duration modification factor: " + minTimeScaleFactor); - logger.debug("maximum duration modification factor: " + maxTimeScaleFactor); - logger.debug("minimum F0 modification factor: " + minPitchScaleFactor); - logger.debug("maximum F0 modification factor: " + maxPitchScaleFactor); - modificationConcatenator = new FdpsolaUnitConcatenator(minTimeScaleFactor, maxTimeScaleFactor, - minPitchScaleFactor, maxPitchScaleFactor); - } catch (Exception e) { - // ignore -- defaults will be used - logger.debug("Initializing FD-PSOLA unit concatenator with default parameter thresholds."); - modificationConcatenator = new FdpsolaUnitConcatenator(); - } - modificationConcatenator.load(database); - } - return modificationConcatenator; - } - - /** - * Gets the domain of this voice - * - * @return the domain - */ - public String getDomain() { - return domain; - } - - public String getExampleText() { - if (exampleText == null) { - return ""; - } else { - return exampleText; - } - } - - public void readExampleText(InputStream in) throws IOException { - BufferedReader reader = new BufferedReader(new InputStreamReader(in, "UTF-8")); - StringBuilder sb = new StringBuilder(); - String line = reader.readLine(); - while (line != null) { - if (!line.startsWith("***")) { - sb.append(line + "\n"); - } - line = reader.readLine(); - } - exampleText = sb.toString(); - } - - public CART[] getF0Trees() { - return f0Carts; - } - - public FeatureDefinition getF0CartsFeatDef() { - if (f0Carts == null || f0Carts.length < 1) - return null; - return f0Carts[0].getFeatureDefinition(); - } - -} diff --git a/marytts-runtime/src/main/java/marytts/unitselection/analysis/HnmVoiceDataDumper.java b/marytts-runtime/src/main/java/marytts/unitselection/analysis/HnmVoiceDataDumper.java deleted file mode 100644 index 5e1d7142e4..0000000000 --- a/marytts-runtime/src/main/java/marytts/unitselection/analysis/HnmVoiceDataDumper.java +++ /dev/null @@ -1,126 +0,0 @@ -/** - * Copyright 2010 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ - -package marytts.unitselection.analysis; - -import java.io.IOException; - -import javax.sound.sampled.AudioFormat; - -import marytts.exceptions.MaryConfigurationException; -import marytts.signalproc.sinusoidal.hntm.analysis.HntmAnalyzerParams; -import marytts.signalproc.sinusoidal.hntm.analysis.HntmSpeechFrame; -import marytts.signalproc.sinusoidal.hntm.analysis.HntmSpeechSignal; -import marytts.signalproc.sinusoidal.hntm.synthesis.HntmSynthesizedSignal; -import marytts.signalproc.sinusoidal.hntm.synthesis.HntmSynthesizer; -import marytts.signalproc.sinusoidal.hntm.synthesis.HntmSynthesizerParams; -import marytts.unitselection.data.HnmDatagram; -import marytts.unitselection.data.HnmTimelineReader; -import marytts.util.data.BufferedDoubleDataSource; -import marytts.util.data.Datagram; -import marytts.util.data.audio.DDSAudioInputStream; -import marytts.util.math.MathUtils; - -/** - * Convenience class to dump relevant data from a HNM unit selection voice to a Praat TextGrid and a wav file for inspection of - * timeline data in external tools (e.g. Praat, WaveSurfer, etc.) - * - * @author steiner - * - */ -public class HnmVoiceDataDumper extends VoiceDataDumper { - - private AudioFormat audioformat; - - public HnmVoiceDataDumper() { - super(); - } - - /** - * {@inheritDoc} - *

- * Also set the audioFormat needed in {@link #getSamples(Datagram[])} - */ - @Override - protected HnmTimelineReader loadAudioTimeline(String fileName) throws IOException, MaryConfigurationException { - HnmTimelineReader audioTimeline = new HnmTimelineReader(fileName); - int sampleRate = audioTimeline.getSampleRate(); - this.audioformat = new AudioFormat(AudioFormat.Encoding.PCM_SIGNED, // encoding - sampleRate, // samples per second - 16, // bits per sample - 1, // mono - 2, // nr. of bytes per frame - sampleRate, // nr. of frames per second - true); // big-endian; - return audioTimeline; - } - - /** - * {@inheritDoc} - *

- * For {@link HnmDatagram}s, the samples must be resynthesized from the HntmSpeechFrame in each HnmDatagram. This requires - * quite a bit of processing. - */ - @Override - protected byte[] getSamples(Datagram[] datagrams) throws IOException { - // init required objects: - HntmSynthesizer hnmSynthesizer = new HntmSynthesizer(); - HntmAnalyzerParams hnmAnalysisParams = new HntmAnalyzerParams(); - HntmSynthesizerParams hnmSynthesisParams = new HntmSynthesizerParams(); - - // get duration from datagrams: - float originalDurationInSeconds = 0; - for (Datagram datagram : datagrams) { - HnmDatagram hnmDatagram = (HnmDatagram) datagram; - originalDurationInSeconds += hnmDatagram.getFrame().deltaAnalysisTimeInSeconds; - } - - // generate HNM signal from frames, correcting the analysis times: - HntmSpeechSignal hnmSpeechSignal = new HntmSpeechSignal(datagrams.length, unitDB.getAudioTimeline().getSampleRate(), - originalDurationInSeconds); - float tAnalysisInSeconds = 0; - for (int i = 0; i < datagrams.length; i++) { - HntmSpeechFrame hnmSpeechFrame = ((HnmDatagram) datagrams[i]).getFrame(); - // correct analysis time: - tAnalysisInSeconds += hnmSpeechFrame.deltaAnalysisTimeInSeconds; - hnmSpeechFrame.tAnalysisInSeconds = tAnalysisInSeconds; - hnmSpeechSignal.frames[i] = hnmSpeechFrame; - } - - // synthesize signal - HntmSynthesizedSignal hnmSynthesizedSignal = hnmSynthesizer.synthesize(hnmSpeechSignal, null, null, null, null, - hnmAnalysisParams, hnmSynthesisParams); - - // scale amplitude: - double[] output = MathUtils.multiply(hnmSynthesizedSignal.output, 1.0 / 32768.0); - - // repack output into byte array: - BufferedDoubleDataSource buffer = new BufferedDoubleDataSource(output); - DDSAudioInputStream audio = new DDSAudioInputStream(buffer, audioformat); - byte[] samples = new byte[(int) audio.getFrameLength() * audioformat.getFrameSize()]; - audio.read(samples); - return samples; - } - - public static void main(String[] args) throws Exception { - new HnmVoiceDataDumper().dumpData(args[0]); - } - -} diff --git a/marytts-runtime/src/main/java/marytts/unitselection/analysis/Phone.java b/marytts-runtime/src/main/java/marytts/unitselection/analysis/Phone.java deleted file mode 100644 index 82575a8876..0000000000 --- a/marytts-runtime/src/main/java/marytts/unitselection/analysis/Phone.java +++ /dev/null @@ -1,748 +0,0 @@ -/** - * Copyright 2010 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ - -package marytts.unitselection.analysis; - -import java.util.Arrays; - -import marytts.modules.phonemiser.Allophone; -import marytts.unitselection.concat.BaseUnitConcatenator.UnitData; -import marytts.unitselection.select.HalfPhoneTarget; -import marytts.unitselection.select.SelectedUnit; -import marytts.util.data.Datagram; -import marytts.util.math.MathUtils; - -import org.apache.commons.lang.ArrayUtils; -import org.w3c.dom.Element; - -/** - * Convenience class containing the selected units and targets of a phone segment, and a host of getters to access their prosodic - * attributes - * - * @author steiner - * - */ -public class Phone { - private HalfPhoneTarget leftTarget; - - private HalfPhoneTarget rightTarget; - - private SelectedUnit leftUnit; - - private SelectedUnit rightUnit; - - private double sampleRate; - - private double[] leftF0Targets; - - private double[] rightF0Targets; - - /** - * Main constructor - * - * @param leftUnit - * which can be null - * @param rightUnit - * which can be null - * @param sampleRate - * of the TimelineReader containing the SelectedUnits, needed to provide duration information - * @throws IllegalArgumentException - * if both the left and right units are null - */ - public Phone(SelectedUnit leftUnit, SelectedUnit rightUnit, int sampleRate) throws IllegalArgumentException { - this.leftUnit = leftUnit; - this.rightUnit = rightUnit; - this.sampleRate = (double) sampleRate; - // targets are extracted from the units for easier access: - try { - this.leftTarget = (HalfPhoneTarget) leftUnit.getTarget(); - } catch (NullPointerException e) { - // leave at null - } - try { - this.rightTarget = (HalfPhoneTarget) rightUnit.getTarget(); - } catch (NullPointerException e) { - if (leftTarget == null) { - throw new IllegalArgumentException("A phone's left and right halves cannot both be null!"); - } else { - // leave at null - } - } - } - - /** - * Get the left selected halfphone unit of this phone - * - * @return the left unit, or null if there is no left unit - */ - public SelectedUnit getLeftUnit() { - return leftUnit; - } - - /** - * Get the right selected halfphone unit of this phone - * - * @return the right unit, or null if there is no right unit - */ - public SelectedUnit getRightUnit() { - return rightUnit; - } - - /** - * Get the left halfphone target of this phone - * - * @return the left target, or null if there is no left target - */ - public HalfPhoneTarget getLeftTarget() { - return leftTarget; - } - - /** - * Get the right halfphone target of this phone - * - * @return the right target, or null if there is no right target - */ - public HalfPhoneTarget getRightTarget() { - return rightTarget; - } - - /** - * Get the datagrams from a SelectedUnit - * - * @param unit - * the SelectedUnit - * @return the datagrams in an array, or null if unit is null - */ - private Datagram[] getUnitFrames(SelectedUnit unit) { - UnitData unitData = getUnitData(unit); - Datagram[] frames = null; - try { - frames = unitData.getFrames(); - } catch (NullPointerException e) { - // leave at null - } - return frames; - } - - /** - * Get this phone's left unit's Datagrams - * - * @return the left unit's Datagrams in an array, or null if there is no left unit - */ - public Datagram[] getLeftUnitFrames() { - return getUnitFrames(leftUnit); - } - - /** - * Get this phone's right unit's Datagrams - * - * @return the right unit's Datagrams in an array, or null if there is no right unit - */ - public Datagram[] getRightUnitFrames() { - return getUnitFrames(rightUnit); - } - - /** - * Get all Datagrams in this phone's units - * - * @return the left and right unit's Datagrams in an array - */ - public Datagram[] getUnitDataFrames() { - Datagram[] leftUnitFrames = getLeftUnitFrames(); - Datagram[] rightUnitFrames = getRightUnitFrames(); - Datagram[] frames = (Datagram[]) ArrayUtils.addAll(leftUnitFrames, rightUnitFrames); - return frames; - } - - /** - * Get the number of Datagrams in a SelectedUnit's UnitData - * - * @param unit - * whose Datagrams to count - * @return the number of Datagrams in the unit, or 0 if unit is null - */ - private int getNumberOfUnitFrames(SelectedUnit unit) { - int numberOfFrames = 0; - try { - Datagram[] frames = getUnitData(unit).getFrames(); - numberOfFrames = frames.length; - } catch (NullPointerException e) { - // leave at 0 - } - return numberOfFrames; - } - - /** - * Get the number of Datagrams in this phone's left unit - * - * @return the number of Datagrams in the left unit, or 0 if there is no left unit - */ - public int getNumberOfLeftUnitFrames() { - return getNumberOfUnitFrames(leftUnit); - } - - /** - * Get the number of Datagrams in this phone's right unit - * - * @return the number of Datagrams in the right unit, or 0 if there is no right unit - */ - public int getNumberOfRightUnitFrames() { - return getNumberOfUnitFrames(rightUnit); - } - - /** - * Get the number of Datagrams in this phone's left and right units - * - * @return the number of Datagrams in this phone - */ - public int getNumberOfFrames() { - return getNumberOfLeftUnitFrames() + getNumberOfRightUnitFrames(); - } - - /** - * Get the durations (in seconds) of each Datagram in this phone's units - * - * @return the left and right unit's Datagram durations (in seconds) in an array - */ - public double[] getFrameDurations() { - Datagram[] frames = getUnitDataFrames(); - double[] durations = new double[frames.length]; - for (int f = 0; f < frames.length; f++) { - durations[f] = frames[f].getDuration() / sampleRate; - } - return durations; - } - - /** - * Get the target duration (in seconds) of a HalfPhoneTarget - * - * @param target - * the target whose duration to get - * @return the target duration in seconds, or 0 if target is null - */ - private double getTargetDuration(HalfPhoneTarget target) { - double duration = 0; - try { - duration = target.getTargetDurationInSeconds(); - } catch (NullPointerException e) { - // leave at 0 - } - return duration; - } - - /** - * Get this phone's left target's duration (in seconds) - * - * @return the left target's duration in seconds, or 0 if there is no left target - */ - public double getLeftTargetDuration() { - return getTargetDuration(leftTarget); - } - - /** - * Get this phone's right target's duration (in seconds) - * - * @return the right target's duration in seconds, or 0 if there is no right target - */ - public double getRightTargetDuration() { - return getTargetDuration(rightTarget); - } - - /** - * Get the predicted duration of this phone (which is the sum of the left and right target's duration) - * - * @return the predicted phone duration in seconds - */ - public double getPredictedDuration() { - return getLeftTargetDuration() + getRightTargetDuration(); - } - - /** - * Convenience getter for a SelectedUnit's UnitData - * - * @param unit - * the unit whose UnitData to get - * @return the UnitData of the unit, or null, if unit is null - */ - private UnitData getUnitData(SelectedUnit unit) { - UnitData unitData = null; - try { - unitData = (UnitData) unit.getConcatenationData(); - } catch (NullPointerException e) { - // leave at null - } - return unitData; - } - - /** - * Get this phone's left unit's UnitData - * - * @return the left unit's UnitData, or null if there is no left unit - */ - public UnitData getLeftUnitData() { - return getUnitData(leftUnit); - } - - /** - * Get this phone's right unit's UnitData - * - * @return the right unit's UnitData, or null if there is no right unit - */ - public UnitData getRightUnitData() { - return getUnitData(rightUnit); - } - - /** - * Get the actual duration (in seconds) of a SelectedUnit, from its UnitData - * - * @param unit - * whose duration to get - * @return the unit's duration in seconds, or 0 if unit is null - */ - private double getUnitDuration(SelectedUnit unit) { - int durationInSamples = 0; - try { - durationInSamples = getUnitData(unit).getUnitDuration(); - } catch (NullPointerException e) { - // leave at 0 - } - double duration = durationInSamples / sampleRate; - return duration; - } - - /** - * Get this phone's left unit's duration (in seconds) - * - * @return the left unit's duration in seconds, or 0 if there is no left unit - */ - public double getLeftUnitDuration() { - return getUnitDuration(leftUnit); - } - - /** - * Get this phone's right unit's duration (in seconds) - * - * @return the right unit's duration in seconds, or 0 if there is no right unit - */ - public double getRightUnitDuration() { - return getUnitDuration(rightUnit); - } - - /** - * Get the realized duration (in seconds) of this phone (which is the sum of the durations of the left and right units) - * - * @return the phone's realized duration in seconds - */ - public double getRealizedDuration() { - return getLeftUnitDuration() + getRightUnitDuration(); - } - - /** - * Get the factor needed to convert the realized duration of a unit to the target duration - * - * @param unit - * whose realized duration to convert - * @param target - * whose duration to match - * @return the multiplication factor to convert from the realized duration to the target duration, or 0 if the unit duration - * is 0 - */ - private double getDurationFactor(SelectedUnit unit, HalfPhoneTarget target) { - double unitDuration = getUnitDuration(unit); - if (unitDuration <= 0) { - // throw new ArithmeticException("Realized duration must be greater than 0!"); - return 0; - } - double targetDuration = getTargetDuration(target); - double durationFactor = targetDuration / unitDuration; - return durationFactor; - } - - /** - * Get the factor to convert this phone's left unit's duration into this phone's left target duration - * - * @return the left duration factor - */ - public double getLeftDurationFactor() { - return getDurationFactor(leftUnit, leftTarget); - } - - /** - * Get the factor to convert this phone's right unit's duration into this phone's right target duration - * - * @return the right duration factor - */ - public double getRightDurationFactor() { - return getDurationFactor(rightUnit, rightTarget); - } - - /** - * Get the duration factors for this phone, one per datagram. Each factor corresponding to a datagram in the left unit is that - * required to convert the left unit's duration to the left target duration, and likewise for the right unit's datagrams. - * - * @return the left and right duration factors for this phone, in an array whose size matched the Datagrams in this phone's - * units - */ - public double[] getFramewiseDurationFactors() { - double[] durationFactors = new double[getNumberOfFrames()]; - int numberOfLeftUnitFrames = getNumberOfLeftUnitFrames(); - double leftDurationFactor = getLeftDurationFactor(); - Arrays.fill(durationFactors, 0, numberOfLeftUnitFrames, leftDurationFactor); - double rightDurationFactor = getRightDurationFactor(); - Arrays.fill(durationFactors, numberOfLeftUnitFrames, getNumberOfFrames(), rightDurationFactor); - return durationFactors; - } - - /** - * Set the target F0 values of this phone's left half, with one value per Datagram in the phone's left unit - * - * @param f0TargetValues - * array of target F0 values to assign to the left halfphone - * @throws IllegalArgumentException - * if the length of f0TargetValues does not match the number of Datagrams in the phone's left unit - */ - public void setLeftTargetF0Values(double[] f0TargetValues) throws IllegalArgumentException { - int numberOfLeftUnitFrames = getNumberOfLeftUnitFrames(); - if (f0TargetValues.length != numberOfLeftUnitFrames) { - throw new IllegalArgumentException("Wrong number of F0 targets (" + f0TargetValues.length - + ") for number of frames (" + numberOfLeftUnitFrames + " in halfphone: '" + leftUnit.toString() + "'"); - } - this.leftF0Targets = f0TargetValues; - } - - /** - * Set the target F0 values of this phone's right half, with one value per Datagram in the phone's right unit - * - * @param f0TargetValues - * array of target F0 values to assign to the right halfphone - * @throws IllegalArgumentException - * if the length of f0TargetValues does not match the number of Datagrams in the phone's right unit - */ - public void setRightTargetF0Values(double[] f0TargetValues) { - if (f0TargetValues.length != getNumberOfRightUnitFrames()) { - throw new IllegalArgumentException("Wrong number of F0 targets (" + f0TargetValues.length - + ") for number of frames (" + getNumberOfRightUnitFrames() + " in halfphone: '" + rightUnit.toString() + "'"); - } - this.rightF0Targets = f0TargetValues; - } - - /** - * Get the target F0 values for this phone's left half, with one value per Datagram in the phone's left unit - * - * @return the target F0 values for the left halfphone in an array - * @throws NullPointerException - * if the target F0 values for the left halfphone are null - */ - public double[] getLeftTargetF0Values() throws NullPointerException { - if (leftF0Targets == null) { - throw new NullPointerException("The left target F0 values have not been assigned!"); - } - return leftF0Targets; - } - - /** - * Get the target F0 values for this phone's right half, with one value per Datagram in the phone's right unit - * - * @return the target F0 values for the right halfphone in an array - * @throws NullPointerException - * if the target F0 values for the right halfphone are null - */ - public double[] getRightTargetF0Values() throws NullPointerException { - if (rightF0Targets == null) { - throw new NullPointerException("The right target F0 values have not been assigned!"); - } - return rightF0Targets; - } - - /** - * Get the target F0 values for this phone, with one value per Datagram in the phone's left and right units - * - * @return the target F0 values with one value per Datagram in an array - */ - public double[] getTargetF0Values() { - double[] f0Targets = ArrayUtils.addAll(leftF0Targets, rightF0Targets); - return f0Targets; - } - - /** - * Get the mean target F0 for this phone - * - * @return the mean predicted F0 value - */ - public double getPredictedF0() { - double meanF0 = MathUtils.mean(getTargetF0Values()); - return meanF0; - } - - /** - * Get the durations (in seconds) of each Datagram in a SelectedUnit's UnitData - * - * @param unit - * whose Datagrams' durations to get - * @return the durations (in seconds) of each Datagram in the unit in an array, or null if unit is null - */ - private double[] getUnitFrameDurations(SelectedUnit unit) { - Datagram[] frames = null; - try { - frames = getUnitData(unit).getFrames(); - } catch (NullPointerException e) { - return null; - } - assert frames != null; - - double[] frameDurations = new double[frames.length]; - for (int f = 0; f < frames.length; f++) { - long frameDuration = frames[f].getDuration(); - frameDurations[f] = frameDuration / sampleRate; // converting to seconds - } - return frameDurations; - } - - /** - * Get the durations (in seconds) of each Datagram in this phone's left unit - * - * @return the durations of each Datagram in the left unit in an array, or null if there is no left unit - */ - public double[] getLeftUnitFrameDurations() { - return getUnitFrameDurations(leftUnit); - } - - /** - * Get the durations (in seconds) of each Datagram in this phone's right unit - * - * @return the durations of each Datagram in the right unit in an array, or null if there is no right unit - */ - public double[] getRightUnitFrameDurations() { - return getUnitFrameDurations(rightUnit); - } - - /** - * Get the durations (in seconds) of each Datagram in this phone's left and right units - * - * @return the durations of all Datagrams in this phone in an array - */ - public double[] getRealizedFrameDurations() { - return ArrayUtils.addAll(getLeftUnitFrameDurations(), getRightUnitFrameDurations()); - } - - /** - * Get the F0 values from a SelectedUnit's Datagrams. - *

- * Since these are not stored explicitly, we are forced to rely on the inverse of the Datagram durations, which means that - * during unvoiced regions, we recover a value of 100 Hz... - * - * @param unit - * from whose Datagrams to recover the F0 values - * @return the F0 value of each Datagram in the unit in an array, or null if the unit is null or any of the Datagrams have - * zero duration - */ - private double[] getUnitF0Values(SelectedUnit unit) { - double[] f0Values = null; - try { - double[] durations = getUnitFrameDurations(unit); - if (!ArrayUtils.contains(durations, 0)) { - f0Values = MathUtils.invert(durations); - } else { - // leave at null - } - } catch (IllegalArgumentException e) { - // leave at null - } - return f0Values; - } - - /** - * Recover the F0 values from this phone's left unit's Datagrams - * - * @return the left unit's F0 values in an array, with one value per Datagram, or null if there is no left unit or any of the - * left unit's Datagrams have zero duration - */ - public double[] getLeftUnitFrameF0s() { - return getUnitF0Values(leftUnit); - } - - /** - * Recover the F0 values from this phone's right unit's Datagrams - * - * @return the right unit's F0 values in an array, with one value per Datagram, or null if there is no right unit or any of - * the right unit's Datagrams have zero duration - */ - public double[] getRightUnitFrameF0s() { - return getUnitF0Values(rightUnit); - } - - /** - * Recover the F0 values from each Datagram in this phone's left and right units - * - * @return the F0 values for each Datagram in this phone, or null if either the left or the right unit contain a Datagram with - * zero duration - */ - public double[] getUnitFrameF0s() { - double[] leftUnitFrameF0s = getLeftUnitFrameF0s(); - double[] rightUnitFrameF0s = getRightUnitFrameF0s(); - double[] unitFrameF0s = ArrayUtils.addAll(leftUnitFrameF0s, rightUnitFrameF0s); - return unitFrameF0s; - } - - /** - * Get the realized F0 by recovering the F0 from all Datagrams in this phone and computing the mean - * - * @return the mean F0 of all Datagrams in this phone's left and right units - */ - public double getRealizedF0() { - double meanF0 = MathUtils.mean(getUnitFrameF0s()); - return meanF0; - } - - /** - * Get the factors required to convert the F0 values recovered from the Datagrams in a SelectedUnit to the target F0 values. - * - * @param unit - * from which to recover the realized F0 values - * @param target - * for which to get the target F0 values - * @return each Datagram's F0 factor in an array, or null if realized and target F0 values differ in length - * @throws ArithmeticException - * if any of the F0 values recovered from the unit's Datagrams is zero - */ - private double[] getUnitF0Factors(SelectedUnit unit, HalfPhoneTarget target) throws ArithmeticException { - double[] unitF0Values = getUnitF0Values(unit); - if (ArrayUtils.contains(unitF0Values, 0)) { - throw new ArithmeticException("Unit frames must not have F0 of 0!"); - } - - double[] targetF0Values; - if (target == null || target.isLeftHalf()) { - targetF0Values = getLeftTargetF0Values(); - } else { - targetF0Values = getRightTargetF0Values(); - } - - double[] f0Factors = null; - try { - f0Factors = MathUtils.divide(targetF0Values, unitF0Values); - } catch (IllegalArgumentException e) { - // leave at null - } - return f0Factors; - } - - /** - * Get the factors to convert each of the F0 values in this phone's left half to the corresponding target value - * - * @return the F0 factors for this phone's left half in an array with one value per Datagram, or null if the number of - * Datagrams does not match the number of left target F0 values - */ - public double[] getLeftF0Factors() { - return getUnitF0Factors(leftUnit, leftTarget); - } - - /** - * Get the factors to convert each of the F0 values in this phone's right half to the corresponding target value - * - * @return the F0 factors for this phone's right half in an array with one value per Datagram, or null if the number of - * Datagrams does not match the number of right target F0 values - */ - public double[] getRightF0Factors() { - return getUnitF0Factors(rightUnit, rightTarget); - } - - /** - * Get the F0 factor for each Datagram in this phone's left and right units - * - * @return the F0 factors, in an array with one value per Datagram - */ - public double[] getF0Factors() { - return ArrayUtils.addAll(getLeftF0Factors(), getRightF0Factors()); - } - - /** - * Get the Allophone represented by this - * - * @return the Allophone - */ - private Allophone getAllophone() { - if (leftTarget != null) { - return leftTarget.getAllophone(); - } else if (rightTarget != null) { - return rightTarget.getAllophone(); - } - return null; - } - - /** - * Determine whether this is a transient phone (i.e. a plosive) - * - * @return true if this is a plosive, false otherwise - */ - public boolean isTransient() { - Allophone allophone = getAllophone(); - if (allophone.isPlosive() || allophone.isAffricate()) { - return true; - } else { - return false; - } - } - - /** - * Determine whether this is a voiced phone - * - * @return true if this is voiced, false otherwise - */ - public boolean isVoiced() { - Allophone allophone = getAllophone(); - if (allophone.isVoiced()) { - return true; - } else { - return false; - } - } - - /** - * get the MaryXML Element corresponding to this Phone's Target - *

- * If both leftTarget and rightTarget are not null, their respective MaryXML Elements should be - * equal. - * - * @return the MaryXML Element, or null if both halfphone targets are null - */ - public Element getMaryXMLElement() { - if (leftTarget != null) { - return leftTarget.getMaryxmlElement(); - } else if (rightTarget != null) { - return rightTarget.getMaryxmlElement(); - } - return null; - } - - /** - * for debugging, provide the names of the left and right targets as the string representation of this class - */ - public String toString() { - String string = ""; - if (leftTarget != null) { - string += " " + leftTarget.getName(); - } - if (rightTarget != null) { - string += " " + rightTarget.getName(); - } - return string; - } -} diff --git a/marytts-runtime/src/main/java/marytts/unitselection/analysis/ProsodyAnalyzer.java b/marytts-runtime/src/main/java/marytts/unitselection/analysis/ProsodyAnalyzer.java deleted file mode 100644 index d09b43901b..0000000000 --- a/marytts-runtime/src/main/java/marytts/unitselection/analysis/ProsodyAnalyzer.java +++ /dev/null @@ -1,437 +0,0 @@ -/** - * Copyright 2010 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ - -package marytts.unitselection.analysis; - -import java.io.File; -import java.io.IOException; -import java.io.PrintWriter; -import java.util.ArrayList; -import java.util.List; -import java.util.ListIterator; - -import marytts.datatypes.MaryXML; -import marytts.modules.acoustic.ProsodyElementHandler; -import marytts.unitselection.select.HalfPhoneTarget; -import marytts.unitselection.select.SelectedUnit; -import marytts.util.MaryUtils; - -import org.apache.commons.lang.ArrayUtils; -import org.apache.log4j.Logger; -import org.w3c.dom.Document; -import org.w3c.dom.Element; -import org.w3c.dom.NodeList; - -/** - * Class to provide high-level, phone-based access to the predicted and realized prosodic parameters in a given unit-selection - * result - * - * @author steiner - * - */ -public class ProsodyAnalyzer { - - private List units; - - private int sampleRate; - - private Logger logger; - - private List phones; - - /** - * Main constructor - *

- * Note that the units are first parsed into phones (and the F0 target values assigned), before any distinction is made - * between those with and without a realized duration (e.g. {@link #getRealizedPhones()}). - * - * @param units - * whose predicted and realized prosody to analyze - * @param sampleRate - * of the unit database, in Hz - * @throws Exception - * if the units cannot be parsed into phones - */ - public ProsodyAnalyzer(List units, int sampleRate) throws Exception { - this.units = units; - this.sampleRate = sampleRate; - - this.logger = MaryUtils.getLogger(this.getClass()); - - // List of phone segments: - this.phones = parseIntoPhones(); - } - - /** - * Parse a list of selected units into the corresponding phone segments - * - * @return List of Phones - * @throws Exception - * if the predicted prosody cannot be determined properly - */ - private List parseIntoPhones() throws Exception { - // initialize List of Phones (note that initial size is not final!): - phones = new ArrayList(units.size() / 2); - // iterate over the units: - int u = 0; - while (u < units.size()) { - // get unit... - SelectedUnit unit = units.get(u); - // ...and its target as a HalfPhoneTarget, so that we can... - HalfPhoneTarget target = (HalfPhoneTarget) unit.getTarget(); - // ...query its position in the phone: - if (target.isLeftHalf()) { - // if this is the left half of a phone... - if (u < units.size() - 1) { - // ...and there is a next unit in the list... - SelectedUnit nextUnit = units.get(u + 1); - HalfPhoneTarget nextTarget = (HalfPhoneTarget) nextUnit.getTarget(); - if (nextTarget.isRightHalf()) { - // ...and the next unit's target is the right half of the phone, add the phone: - phones.add(new Phone(unit, nextUnit, sampleRate)); - u++; - } else { - // otherwise, add a degenerate phone with no right halfphone: - phones.add(new Phone(unit, null, sampleRate)); - } - } else { - // otherwise, add a degenerate phone with no right halfphone: - phones.add(new Phone(unit, null, sampleRate)); - } - } else { - // otherwise, add a degenerate phone with no left halfphone: - phones.add(new Phone(null, unit, sampleRate)); - } - u++; - } - - // make sure we've seen all the units: - assert u == units.size(); - - // assign target F0 values to Phones: - insertTargetF0Values(); - - return phones; - } - - /** - * Assign predicted F0 values to the phones by parsing the XML Document - * - * @throws Exception - * if the Document cannot be accessed - */ - private void insertTargetF0Values() throws Exception { - NodeList phoneNodes; - try { - phoneNodes = getPhoneNodes(); - } catch (Exception e) { - throw new Exception("Could not get the phone Nodes from the Document", e); - } - - // count the number of Datagrams we need, which is the number of F0 target values the ProsodyElementHandler will return: - int totalNumberOfFrames = getNumberOfFrames(); - - // this method hinges on the F0 attribute parsing done in modules.acoustic - ProsodyElementHandler elementHandler = new ProsodyElementHandler(); - double[] f0Targets = elementHandler.getF0Contour(phoneNodes, totalNumberOfFrames); - - int f0TargetStartIndex = 0; - for (Phone phone : phones) { - int numberOfLeftUnitFrames = phone.getNumberOfLeftUnitFrames(); - int f0TargetMidIndex = f0TargetStartIndex + numberOfLeftUnitFrames; - double[] leftF0Targets = ArrayUtils.subarray(f0Targets, f0TargetStartIndex, f0TargetMidIndex); - phone.setLeftTargetF0Values(leftF0Targets); - - int numberOfRightUnitFrames = phone.getNumberOfRightUnitFrames(); - int f0TargetEndIndex = f0TargetMidIndex + numberOfRightUnitFrames; - double[] rightF0Targets = ArrayUtils.subarray(f0Targets, f0TargetMidIndex, f0TargetEndIndex); - phone.setRightTargetF0Values(rightF0Targets); - - f0TargetStartIndex = f0TargetEndIndex; - } - return; - } - - /** - * Get the List of Phones - * - * @return the Phones - */ - public List getPhones() { - return phones; - } - - /** - * Get the List of Phones that have a predicted duration greater than zero - * - * @return the List of realized Phones - */ - public List getRealizedPhones() { - List realizedPhones = new ArrayList(phones.size()); - for (Phone phone : phones) { - if (phone.getPredictedDuration() > 0) { - realizedPhones.add(phone); - } - } - return realizedPhones; - } - - /** - * Get NodeList for Phones from Document - * - * @return NodeList of Phones - * @throws Exception - * if Document cannot be accessed - */ - private NodeList getPhoneNodes() throws Exception { - Document document = getDocument(); - NodeList phoneNodes; - try { - phoneNodes = document.getElementsByTagName(MaryXML.PHONE); - } catch (NullPointerException e) { - throw new Exception("Could not access the Document!", e); - } - return phoneNodes; - } - - /** - * For the first phone with a MaryXMLElement we encounter, return that Element's Document - * - * @return the Document containing the {@link #phones} or null if no phone is able to provide a MaryXMLElement - */ - private Document getDocument() { - for (Phone phone : phones) { - Element phoneElement = phone.getMaryXMLElement(); - if (phoneElement != null) { - return phoneElement.getOwnerDocument(); - } - } - return null; - } - - /** - * Get the number of Datagrams in all Phones - * - * @return the number of Datagrams in all Phones - */ - private int getNumberOfFrames() { - int totalNumberOfFrames = 0; - for (Phone phone : phones) { - totalNumberOfFrames += phone.getNumberOfFrames(); - } - return totalNumberOfFrames; - } - - /** - * Get duration factors representing ratio of predicted and realized halfphone Unit durations. Units with zero predicted or - * realized duration receive a factor of 0. - * - * @return List of duration factors - */ - public List getDurationFactors() { - // list of duration factors, one per halfphone unit: - List durationFactors = new ArrayList(units.size()); - - // iterate over phone segments: - for (Phone phone : phones) { - double leftDurationFactor = phone.getLeftDurationFactor(); - if (leftDurationFactor > 0) { - durationFactors.add(leftDurationFactor); - logger.debug("duration factor for unit " + phone.getLeftUnit().getTarget().getName() + " -> " - + leftDurationFactor); - } - double rightDurationFactor = phone.getRightDurationFactor(); - if (rightDurationFactor > 0) { - // ...add the duration factor to the list: - durationFactors.add(rightDurationFactor); - logger.debug("duration factor for unit " + phone.getRightUnit().getTarget().getName() + " -> " - + rightDurationFactor); - } - } - - return durationFactors; - } - - /* - * Some ad-hoc methods for HnmUnitConcatenator: - */ - - public double[] getDurationFactorsFramewise() { - double[] f0Factors = null; - for (Phone phone : phones) { - double[] phoneF0Factors = phone.getFramewiseDurationFactors(); - f0Factors = ArrayUtils.addAll(f0Factors, phoneF0Factors); - } - return f0Factors; - } - - public double[] getFrameMidTimes() { - double[] frameDurations = null; - for (Phone phone : phones) { - double[] phoneFrameDurations = phone.getFrameDurations(); - frameDurations = ArrayUtils.addAll(frameDurations, phoneFrameDurations); - } - - assert frameDurations != null; - double[] frameMidTimes = new double[frameDurations.length]; - double frameStartTime = 0; - for (int f = 0; f < frameDurations.length; f++) { - frameMidTimes[f] = frameStartTime + frameDurations[f] / 2; - frameStartTime += frameDurations[f]; - } - - return frameMidTimes; - } - - public double[] getF0Factors() { - double[] f0Factors = null; - for (Phone phone : phones) { - double[] phoneF0Factors = phone.getF0Factors(); - f0Factors = ArrayUtils.addAll(f0Factors, phoneF0Factors); - } - return f0Factors; - } - - /** - * For debugging, generate Praat DurationTier, which can be used for PSOLA-based manipulation in Praat. - *

- * Notes: - *

    - *
  • Initial silence is skipped.
    • - *
  • Units with zero realized duration are ignored.
    • - *
  • To avoid gradual interpolation between points, place two points around each unit boundary, separated by - * MIN_SKIP; this workaround allows one constant factor per unit.
    • - *
- * - * @param fileName - * of the DurationTier to be generated - * @throws IOException - */ - public void writePraatDurationTier(String fileName) throws IOException { - - // initialize times and values with a size corresponding to two elements (start and end) per unit: - ArrayList times = new ArrayList(units.size() * 2); - ArrayList values = new ArrayList(units.size() * 2); - - final double MIN_SKIP = 1e-15; - - // cumulative time pointer: - double time = 0; - - // iterate over phones, skipping the initial silence: - // TODO is this really robust? - ListIterator phoneIterator = phones.listIterator(1); - while (phoneIterator.hasNext()) { - Phone phone = phoneIterator.next(); - - // process left halfphone unit: - if (phone.getLeftUnitDuration() > 0) { - // add point at unit start: - times.add(time); - values.add(phone.getLeftDurationFactor()); - - // increment time pointer by unit duration: - time += phone.getLeftUnitDuration(); - - // add point at unit end: - times.add(time - MIN_SKIP); - values.add(phone.getLeftDurationFactor()); - } - // process right halfphone unit: - if (phone.getRightUnitDuration() > 0) { - // add point at unit start: - times.add(time); - values.add(phone.getRightDurationFactor()); - - // increment time pointer by unit duration: - time += phone.getRightUnitDuration(); - - // add point at unit end: - times.add(time - MIN_SKIP); - values.add(phone.getRightDurationFactor()); - } - } - - // open file for writing: - File durationTierFile = new File(fileName); - PrintWriter out = new PrintWriter(durationTierFile); - - // print header: - out.println("\"ooTextFile\""); - out.println("\"DurationTier\""); - out.println(String.format("0 %f %d", time, times.size())); - - // print points (times and values): - for (int i = 0; i < times.size(); i++) { - // Note: time precision should be greater than MIN_SKIP: - out.println(String.format("%.16f %f", times.get(i), values.get(i))); - } - - // flush and close: - out.close(); - } - - /** - * For debugging, generate Praat PitchTier, which can be used for PSOLA-based manipulation in Praat. - * - * @param fileName - * of the PitchTier to be generated - * @throws IOException - */ - public void writePraatPitchTier(String fileName) throws IOException { - - // initialize times and values: - ArrayList times = new ArrayList(); - ArrayList values = new ArrayList(); - - // cumulative time pointer: - double time = 0; - - // iterate over phones, skipping the initial silence: - ListIterator phoneIterator = phones.listIterator(1); - while (phoneIterator.hasNext()) { - Phone phone = phoneIterator.next(); - double[] frameTimes = phone.getRealizedFrameDurations(); - double[] frameF0s = phone.getUnitFrameF0s(); - for (int f = 0; f < frameF0s.length; f++) { - time += frameTimes[f]; - times.add(time); - values.add(frameF0s[f]); - } - } - - // open file for writing: - File durationTierFile = new File(fileName); - PrintWriter out = new PrintWriter(durationTierFile); - - // print header: - out.println("\"ooTextFile\""); - out.println("\"PitchTier\""); - out.println(String.format("0 %f %d", time, times.size())); - - // print points (times and values): - for (int i = 0; i < times.size(); i++) { - out.println(String.format("%.16f %f", times.get(i), values.get(i))); - } - - // flush and close: - out.close(); - } -} diff --git a/marytts-runtime/src/main/java/marytts/unitselection/analysis/VoiceDataDumper.java b/marytts-runtime/src/main/java/marytts/unitselection/analysis/VoiceDataDumper.java deleted file mode 100644 index f1b1b033a5..0000000000 --- a/marytts-runtime/src/main/java/marytts/unitselection/analysis/VoiceDataDumper.java +++ /dev/null @@ -1,351 +0,0 @@ -/** - * Copyright 2010 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ - -package marytts.unitselection.analysis; - -import java.io.BufferedOutputStream; -import java.io.BufferedWriter; -import java.io.ByteArrayOutputStream; -import java.io.DataOutputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.PrintWriter; -import java.nio.BufferUnderflowException; - -import marytts.exceptions.MaryConfigurationException; -import marytts.features.FeatureDefinition; -import marytts.features.FeatureVector; -import marytts.server.MaryProperties; -import marytts.unitselection.data.FeatureFileReader; -import marytts.unitselection.data.TimelineReader; -import marytts.unitselection.data.Unit; -import marytts.unitselection.data.UnitDatabase; -import marytts.unitselection.data.UnitFileReader; -import marytts.util.data.Datagram; -import marytts.util.data.text.PraatInterval; -import marytts.util.data.text.PraatIntervalTier; -import marytts.util.data.text.PraatTextGrid; - -/** - * Convenience class to dump relevant data from a unit selection voice to a Praat TextGrid and a wav file for inspection of - * timeline data in external tools (e.g. Praat, WaveSurfer, etc.) - * - * @author steiner - * - */ -public class VoiceDataDumper { - protected UnitDatabase unitDB; - - protected FeatureFileReader featureFileReader; - - protected long numSamples = 0; - - protected FeatureDefinition featureDefinition; - - protected int phoneFeatureIndex; - - protected int halfphoneLRFeatureIndex; - - public VoiceDataDumper() { - - } - - /** - * @see marytts.util.data.audio.WavWriter#byteswap(int) - */ - protected int byteswap(int val) { - return (((val & 0xff000000) >>> 24) + ((val & 0x00ff0000) >>> 8) + ((val & 0x0000ff00) << 8) + ((val & 0x000000ff) << 24)); - } - - /** - * @see marytts.util.data.audio.WavWriter#byteswap(short) - */ - protected short byteswap(short val) { - return ((short) ((((int) (val) & 0xff00) >>> 8) + (((int) (val) & 0x00ff) << 8))); - } - - /** - * Load audio timeline from file - * - * @param fileName - * to load - * @return TimelineReader - * @throws IOException - */ - protected TimelineReader loadAudioTimeline(String fileName) throws IOException, MaryConfigurationException { - return new TimelineReader(fileName); - } - - /** - * Load unit database from various relevant files - * - * @param audioTimelineFileName - * to load - * @param basenameTimelineFileName - * to load - * @param unitFileName - * to load - * @throws IOException - */ - protected void loadUnitDatabase(String audioTimelineFileName, String basenameTimelineFileName, String unitFileName) - throws IOException, MaryConfigurationException { - unitDB = new UnitDatabase(); - UnitFileReader unitFileReader = new UnitFileReader(unitFileName); - TimelineReader audioTimelineReader = loadAudioTimeline(audioTimelineFileName); - TimelineReader basenameTimelineReader = new TimelineReader(basenameTimelineFileName); - unitDB.load(null, null, unitFileReader, null, audioTimelineReader, basenameTimelineReader, 0); - } - - /** - * Load unit feature file from file - * - * @param fileName - * to load - * @throws IOException - */ - protected void loadFeatureFile(String fileName) throws IOException, MaryConfigurationException { - featureFileReader = new FeatureFileReader(fileName); - featureDefinition = featureFileReader.getFeatureDefinition(); - phoneFeatureIndex = featureDefinition.getFeatureIndex("phone"); - halfphoneLRFeatureIndex = featureDefinition.getFeatureIndex("halfphone_lr"); - } - - /** - * Get total duration of a Datagram array - * - * @param datagrams - * whose duration to get - * @return total duration in seconds - */ - protected double getDuration(Datagram[] datagrams) { - double totalDuration = 0; - for (Datagram datagram : datagrams) { - totalDuration += datagram.getDuration() / (float) unitDB.getAudioTimeline().getSampleRate(); - } - return totalDuration; - } - - /** - * Get raw samples from all Datagrams in an array - * - * @param datagrams - * whose samples to get - * @return raw samples as stored in the Datagrams - * @throws IOException - */ - protected byte[] getSamples(Datagram[] datagrams) throws IOException { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - for (Datagram datagram : datagrams) { - byte[] data = datagram.getData(); - baos.write(data); - } - byte[] samples = baos.toByteArray(); - return samples; - } - - /** - * Dump units to Praat TextGrid. This will have three tiers: - *
    - *
  1. halfphone units, labeled with unit indices;
    2. - *
  2. phone units, labeled with allophones;
    3. - *
  3. basenames, labeled with basename of original utterance.
    4. - *
- * - * @param fileName - * of new TextGrid - * @throws IOException - * if data files cannot be read, or TextGrid cannot be written - */ - protected void dumpTextGrid(String fileName) throws IOException { - // init the tiers: - PraatIntervalTier unitTier = new PraatIntervalTier("unitindex"); - PraatIntervalTier phoneTier = new PraatIntervalTier("halfphone"); - PraatIntervalTier basenameTier = new PraatIntervalTier("basename"); - - // init some variables: - double prevHalfPhoneUnitDurationInSeconds = 0; - double basenameDurationInSeconds = 0; - String basenameLabel = null; - - // iterate over all units: - for (int unitIndex = 0; unitIndex < unitDB.getUnitFileReader().getNumberOfUnits(); unitIndex++) { - // if (unitIndex > 727) { - // break; - // } - Unit unit = unitDB.getUnitFileReader().getUnit(unitIndex); - if (unit.isEdgeUnit()) { - // if this is the left edge, basenameDurationInSeconds will be 0 - if (basenameDurationInSeconds > 0) { - // add basename interval - PraatInterval basenameInterval = new PraatInterval(basenameDurationInSeconds, basenameLabel); - basenameTier.appendInterval(basenameInterval); - basenameDurationInSeconds = 0; - } - continue; // ignore edge units (also, avoid ticket:335) - } - - // iterate over datagrams to get exact duration: - Datagram[] datagrams; - try { - datagrams = unitDB.getAudioTimeline().getDatagrams(unit, unitDB.getAudioTimeline().getSampleRate()); - } catch (BufferUnderflowException e) { - throw e; - } - double halfPhoneUnitDurationInSeconds = getDuration(datagrams); - // cumulative sample count for wav file header: - byte[] buf = getSamples(datagrams); - numSamples += buf.length; - - // keep track of basename duration and label: - basenameDurationInSeconds += halfPhoneUnitDurationInSeconds; - basenameLabel = unitDB.getFilename(unit); - - // halfphone unit interval (labeled with unit index): - PraatInterval interval = new PraatInterval(halfPhoneUnitDurationInSeconds, Integer.toString(unit.index)); - unitTier.appendInterval(interval); - - // lazy way of checking that we have both halves of the phone: - FeatureVector features = featureFileReader.getFeatureVector(unit); - String halfphoneLR = features.getFeatureAsString(halfphoneLRFeatureIndex, featureDefinition); - if (halfphoneLR.equals("R")) { - // phone interval: - double phoneUnitDurationInSeconds = halfPhoneUnitDurationInSeconds + prevHalfPhoneUnitDurationInSeconds; - String phoneLabel = features.getFeatureAsString(phoneFeatureIndex, featureDefinition); - PraatInterval phoneInterval = new PraatInterval(phoneUnitDurationInSeconds, phoneLabel); - phoneTier.appendInterval(phoneInterval); - } - prevHalfPhoneUnitDurationInSeconds = halfPhoneUnitDurationInSeconds; - } - - // update time domains: - unitTier.updateBoundaries(); - phoneTier.updateBoundaries(); - basenameTier.updateBoundaries(); - - // create TextGrid: - PraatTextGrid textGrid = new PraatTextGrid(); - textGrid.appendTier(unitTier); - textGrid.appendTier(phoneTier); - textGrid.appendTier(basenameTier); - - // write to text file: - BufferedWriter output = new BufferedWriter(new PrintWriter(fileName)); - output.write(textGrid.toString()); - output.close(); - } - - /** - * Adapted from {@link marytts.util.data.audio.WavWriter#export(String, int, byte[])} and - * {@link marytts.util.data.audio.WavWriter#doWrite(String, int)} - */ - protected void dumpAudio(String fileName) throws IOException { - // refuse to write wav file if we don't know how many samples there are: - if (!(numSamples > 0)) { - return; - } - - // open wav file, and write header: - DataOutputStream dos = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(fileName))); - int nBytesPerSample = 2; - dos.writeBytes("RIFF"); // "RIFF" in ascii - dos.writeInt(byteswap((int) (36 + numSamples))); // Chunk size - dos.writeBytes("WAVEfmt "); - dos.writeInt(byteswap(16)); // chunk size, 16 for PCM - dos.writeShort(byteswap((short) 1)); // PCM format - dos.writeShort(byteswap((short) 1)); // Mono, one channel - dos.writeInt(byteswap(unitDB.getAudioTimeline().getSampleRate())); // Samplerate - dos.writeInt(byteswap(unitDB.getAudioTimeline().getSampleRate() * nBytesPerSample)); // Byte-rate - dos.writeShort(byteswap((short) (nBytesPerSample))); // Nbr of bytes per samples x nbr of channels - dos.writeShort(byteswap((short) (nBytesPerSample * 8))); // nbr of bits per sample - dos.writeBytes("data"); - dos.writeInt(byteswap((int) numSamples)); - - // implicitly unit-wise buffered writing of samples: - for (int unitIndex = 0; unitIndex < unitDB.getUnitFileReader().getNumberOfUnits(); unitIndex++) { - Unit unit = unitDB.getUnitFileReader().getUnit(unitIndex); - if (unit.isEdgeUnit()) { - continue; // ignore edge units (also, avoid ticket:335) - } - - Datagram[] datagrams = unitDB.getAudioTimeline().getDatagrams(unit, unitDB.getAudioTimeline().getSampleRate()); - byte[] buf = getSamples(datagrams); - - // write buffer to file: - // Byte-swap the samples - byte b = 0; - for (int j = 0; j < buf.length - 1; j += 2) { - b = buf[j]; - try { - buf[j] = buf[j + 1]; - } catch (ArrayIndexOutOfBoundsException e) { - throw e; - } - buf[j + 1] = b; - } - dos.write(buf); - } - - dos.close(); - } - - /** - * Get file names from voice config file. Dump relevant data from audio timeline, unit file, etc. to Praat TextGrid and wav - * file. - * - * @param voiceName - * for config file to read (e.g. "bits3") - * @throws Exception - */ - protected void dumpData(String voiceName) throws Exception { - - String audioTimelineFileName = MaryProperties.needFilename("voice." + voiceName + ".audioTimelineFile"); - String basenameTimelineFileName = MaryProperties.needFilename("voice." + voiceName + ".basenameTimeline"); - String unitFileName = MaryProperties.needFilename("voice." + voiceName + ".unitsFile"); - String featureFileName = MaryProperties.needFilename("voice." + voiceName + ".featureFile"); - String textGridFilename = audioTimelineFileName.replace(".mry", ".TextGrid"); - String wavFilename = audioTimelineFileName.replace(".mry", ".wav"); - - loadUnitDatabase(audioTimelineFileName, basenameTimelineFileName, unitFileName); - loadFeatureFile(featureFileName); - System.out.println("All files loaded."); - dumpTextGrid(textGridFilename); - System.out.println("Dumped TextGrid to " + textGridFilename); - dumpAudio(wavFilename); - System.out.println("Dumped audio to " + wavFilename); - } - - /** - * Main method. Add VOICE jar to classpath, then call with - * - * 
-	 * -ea -Xmx1gb -Dmary.base=$MARYBASE VOICE
-	 *
- * - * or something similar - * - * @param args - * voice name (without the Locale) of voice to dump data from - * @throws Exception - */ - public static void main(String[] args) throws Exception { - new VoiceDataDumper().dumpData(args[0]); - } - -} diff --git a/marytts-runtime/src/main/java/marytts/unitselection/concat/BaseUnitConcatenator.java b/marytts-runtime/src/main/java/marytts/unitselection/concat/BaseUnitConcatenator.java deleted file mode 100644 index e708236035..0000000000 --- a/marytts-runtime/src/main/java/marytts/unitselection/concat/BaseUnitConcatenator.java +++ /dev/null @@ -1,315 +0,0 @@ -/** - * Portions Copyright 2006 DFKI GmbH. - * Portions Copyright 2001 Sun Microsystems, Inc. - * Portions Copyright 1999-2001 Language Technologies Institute, - * Carnegie Mellon University. - * All Rights Reserved. Use is subject to license terms. - * - * Permission is hereby granted, free of charge, to use and distribute - * this software and its documentation without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of this work, and to - * permit persons to whom this work is furnished to do so, subject to - * the following conditions: - * - * 1. The code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * 2. Any modifications must be clearly marked as such. - * 3. Original authors' names are not deleted. - * 4. The authors' names are not used to endorse or promote products - * derived from this software without specific prior written - * permission. - * - * DFKI GMBH AND THE CONTRIBUTORS TO THIS WORK DISCLAIM ALL WARRANTIES WITH - * REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL DFKI GMBH NOR THE - * CONTRIBUTORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL - * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR - * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS - * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF - * THIS SOFTWARE. - */ -package marytts.unitselection.concat; - -import java.io.IOException; -import java.util.Arrays; -import java.util.LinkedList; -import java.util.List; - -import javax.sound.sampled.AudioFormat; -import javax.sound.sampled.AudioInputStream; - -import marytts.unitselection.analysis.ProsodyAnalyzer; -import marytts.unitselection.data.TimelineReader; -import marytts.unitselection.data.UnitDatabase; -import marytts.unitselection.select.SelectedUnit; -import marytts.util.MaryUtils; -import marytts.util.data.BufferedDoubleDataSource; -import marytts.util.data.Datagram; -import marytts.util.data.DatagramDoubleDataSource; -import marytts.util.data.DoubleDataSource; -import marytts.util.data.audio.DDSAudioInputStream; - -import org.apache.log4j.Logger; - -/** - * Concatenates Units and returns an audio stream - * - * - */ -public class BaseUnitConcatenator implements UnitConcatenator { - protected Logger logger; - protected UnitDatabase database; - protected TimelineReader timeline; - protected AudioFormat audioformat; - protected double unitToTimelineSampleRateFactor; - - protected ProsodyAnalyzer prosodyAnalyzer; - - /** - * Empty Constructor; need to call load(UnitDatabase) separately - * - * @see #load(UnitDatabase) - */ - public BaseUnitConcatenator() { - logger = MaryUtils.getLogger(this.getClass()); - } - - public void load(UnitDatabase unitDatabase) { - this.database = unitDatabase; - this.timeline = database.getAudioTimeline(); - int sampleRate = timeline.getSampleRate(); - this.audioformat = new AudioFormat(AudioFormat.Encoding.PCM_SIGNED, sampleRate, // samples per second - 16, // bits per sample - 1, // mono - 2, // nr. of bytes per frame - sampleRate, // nr. of frames per second - true); // big-endian; - this.unitToTimelineSampleRateFactor = sampleRate / (double) database.getUnitFileReader().getSampleRate(); - } - - /** - * Provide the audio format which will be produced by this unit concatenator. - * - * @return the audio format - */ - public AudioFormat getAudioFormat() { - return audioformat; - } - - /** - * Build the audio stream from the units - * - * @param units - * the units - * @return the resulting audio stream - */ - public AudioInputStream getAudio(List units) throws IOException { - logger.debug("Getting audio for " + units.size() + " units"); - - // 1. Get the raw audio material for each unit from the timeline - getDatagramsFromTimeline(units); - - // 2. Determine target pitchmarks (= duration and f0) for each unit - determineTargetPitchmarks(units); - - // 2a. Analyze SelectedUnits wrt predicted vs. realized prosody - try { - prosodyAnalyzer = new ProsodyAnalyzer(units, timeline.getSampleRate()); - } catch (Exception e) { - throw new IOException("Could not analyze prosody!", e); - } - - // 3. Generate audio to match the target pitchmarks as closely as possible - return generateAudioStream(units); - } - - /** - * Get the raw audio material for each unit from the timeline. - * - * @param units - */ - protected void getDatagramsFromTimeline(List units) throws IOException { - for (SelectedUnit unit : units) { - UnitData unitData = new UnitData(); - unit.setConcatenationData(unitData); - int nSamples = 0; - int unitSize = unitToTimeline(unit.getUnit().duration); // convert to timeline samples - long unitStart = unitToTimeline(unit.getUnit().startTime); // convert to timeline samples - // System.out.println("Unit size "+unitSize+", pitchmarksInUnit "+pitchmarksInUnit); - Datagram[] datagrams = timeline.getDatagrams(unitStart, (long) unitSize); - unitData.setFrames(datagrams); - } - } - - /** - * Determine target pitchmarks (= duration and f0) for each unit. - * - * @param units - */ - protected void determineTargetPitchmarks(List units) { - for (SelectedUnit unit : units) { - UnitData unitData = (UnitData) unit.getConcatenationData(); - assert unitData != null : "Should not have null unitdata here"; - Datagram[] datagrams = unitData.getFrames(); - Datagram[] frames = null; // frames to realise - // The number and duration of the frames to realise - // must be the result of the target pitchmark computation. - if (datagrams != null && datagrams.length > 0) { - frames = datagrams; - } else { // no datagrams -- set as silence - int targetLength = (int) (unit.getTarget().getTargetDurationInSeconds() * timeline.getSampleRate()); - frames = new Datagram[] { createZeroDatagram(targetLength) }; - } - int unitDuration = 0; - for (int i = 0; i < frames.length; i++) { - int dur = (int) frames[i].getDuration(); - unitDuration += frames[i].getDuration(); - } - - unitData.setUnitDuration(unitDuration); - unitData.setFrames(frames); - } - } - - /** - * Generate audio to match the target pitchmarks as closely as possible. - * - * @param units - * @return - * @throws IOException - */ - protected AudioInputStream generateAudioStream(List units) throws IOException { - LinkedList datagrams = new LinkedList(); - for (SelectedUnit unit : units) { - UnitData unitData = (UnitData) unit.getConcatenationData(); - assert unitData != null : "Should not have null unitdata here"; - Datagram[] frames = unitData.getFrames(); - assert frames != null : "Cannot generate audio from null frames"; - // Generate audio from frames - datagrams.addAll(Arrays.asList(frames)); - } - - DoubleDataSource audioSource = new DatagramDoubleDataSource(datagrams); - return new DDSAudioInputStream(new BufferedDoubleDataSource(audioSource), audioformat); - } - - /** - * Create a datagram appropriate for this unit concatenator which contains only zero values as samples. - * - * @param length - * the number of zeros that the datagram should contain - * @return - */ - protected Datagram createZeroDatagram(int length) { - return new Datagram(length, new byte[2 * length]); - } - - protected int unitToTimeline(int duration) { - return (int) (duration * unitToTimelineSampleRateFactor); - } - - protected long unitToTimeline(long time) { - return (long) (time * unitToTimelineSampleRateFactor); - } - - public static class UnitData { - protected int[] pitchmarks; - protected Datagram[] frames; - protected Datagram rightContextFrame; - - protected int unitDuration = -1; - - public UnitData() { - } - - /** - * Set the array of to-be-realised pitchmarks for the realisation of the selected unit. - * - * @param pitchmarks - */ - // TODO why is this never used? - public void setPitchmarks(int[] pitchmarks) { - this.pitchmarks = pitchmarks; - } - - public int[] getPitchmarks() { - return pitchmarks; - } - - /** - * Get the pitchmark marking the end of the period with the index number periodIndex. - * - * @param periodIndex - * @return the pitchmark position, in samples - */ - public int getPitchmark(int periodIndex) { - return pitchmarks[periodIndex]; - } - - /** - * Get the length of the pitch period ending with pitchmark with the index number periodIndex. - * - * @param periodIndex - * @return the period length, in samples - */ - public int getPeriodLength(int periodIndex) { - if (0 <= periodIndex && periodIndex < pitchmarks.length) { - if (periodIndex > 0) { - return pitchmarks[periodIndex] - pitchmarks[periodIndex - 1]; - } else { - return pitchmarks[periodIndex]; - } - } else { - return 0; - } - } - - public int getNumberOfPitchmarks() { - return pitchmarks.length; - } - - public void setFrames(Datagram[] frames) { - this.frames = frames; - } - - public Datagram[] getFrames() { - return frames; - } - - public void setFrame(int frameIndex, Datagram frame) { - this.frames[frameIndex] = frame; - } - - public Datagram getFrame(int frameIndex) { - return frames[frameIndex]; - } - - public void setRightContextFrame(Datagram aRightContextFrame) { - this.rightContextFrame = aRightContextFrame; - } - - public Datagram getRightContextFrame() { - return rightContextFrame; - } - - /** - * Set the realised duration of this unit, in samples. - * - * @param duration - */ - public void setUnitDuration(int duration) { - this.unitDuration = duration; - } - - /** - * Get the realised duration of this unit, in samples - * - * @return - */ - public int getUnitDuration() { - return unitDuration; - } - - } -} diff --git a/marytts-runtime/src/main/java/marytts/unitselection/concat/DatagramOverlapDoubleDataSource.java b/marytts-runtime/src/main/java/marytts/unitselection/concat/DatagramOverlapDoubleDataSource.java deleted file mode 100644 index 4e19796697..0000000000 --- a/marytts-runtime/src/main/java/marytts/unitselection/concat/DatagramOverlapDoubleDataSource.java +++ /dev/null @@ -1,148 +0,0 @@ -/** - * Copyright 2004-2006 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.unitselection.concat; - -import marytts.signalproc.window.DynamicTwoHalvesWindow; -import marytts.signalproc.window.Window; -import marytts.util.data.BufferedDoubleDataSource; -import marytts.util.data.Datagram; -import marytts.util.data.DoubleDataSource; - -public class DatagramOverlapDoubleDataSource extends BufferedDoubleDataSource { - protected Datagram[][] datagrams; - protected Datagram[] rightContexts; - protected int p; // point to current datagrams/rightContext - protected int q; // point to current datagram within datagrams[p] - protected int totalRead; // count samples read from datagrams - - /** - * Construct an double data source from the given array of datagram arrays and right contexts. - * - * @param datagrams - */ - public DatagramOverlapDoubleDataSource(Datagram[][] datagrams, Datagram[] rightContexts) { - super((DoubleDataSource) null); - this.datagrams = datagrams; - this.rightContexts = rightContexts; - dataLength = 0; - for (int i = 0; i < datagrams.length; i++) { - for (int j = 0; j < datagrams[i].length; j++) { - dataLength += datagrams[i][j].getDuration(); - } - } - p = 0; - q = 0; - } - - /** - * Whether or not any more data can be read from this data source. - * - * @return true if another call to getData() will return data, false otherwise. - */ - public boolean hasMoreData() { - if (currentlyInBuffer() > 0 || totalRead < dataLength) - return true; - return false; - } - - /** - * The number of doubles that can currently be read from this double data source without blocking. This number can change over - * time. - * - * @return the number of doubles that can currently be read without blocking - */ - public int available() { - int available = (int) (currentlyInBuffer() + dataLength - totalRead); - return available; - } - - /** - * Attempt to get more data from the input source. If less than this can be read, the possible amount will be read, but - * canReadMore() will return false afterwards. - * - * @param minLength - * the amount of data to get from the input source - * @return true if the requested amount could be read, false if none or less data could be read. - */ - protected boolean readIntoBuffer(int minLength) { - if (bufferSpaceLeft() < minLength) { - // current buffer cannot hold the data requested; - // need to make it larger - increaseBufferSize(minLength + currentlyInBuffer()); - } else if (buf.length - writePos < minLength) { - compact(); // create a contiguous space for the new data - } - // Now we have a buffer that can hold at least minLength new data points - int readSum = 0; - // read blocks: - - while (readSum < minLength && p < datagrams.length) { - if (q >= datagrams[p].length) { - p++; - q = 0; - } else { - Datagram next = datagrams[p][q]; - int length = (int) next.getDuration(); - // System.out.println("Unit duration = " + String.valueOf(length)); - if (buf.length < writePos + length) { - increaseBufferSize(writePos + length); - } - int read = readDatagram(next, buf, writePos); - if (q == 0 && p > 0 && rightContexts[p - 1] != null) { - // overlap-add situation - // window the data that we have just read with the left half of a HANN window: - new DynamicTwoHalvesWindow(Window.HANNING).applyInlineLeftHalf(buf, writePos, read); - // and overlap-add the previous right context, windowed with the right half of a HANN window: - double[] context = new double[(int) rightContexts[p - 1].getDuration()]; - readDatagram(rightContexts[p - 1], context, 0); - new DynamicTwoHalvesWindow(Window.HANNING).applyInlineRightHalf(context, 0, context.length); - for (int i = 0, iMax = Math.min(read, context.length); i < iMax; i++) { - buf[writePos + i] += context[i]; - } - } - writePos += read; - readSum += read; - totalRead += read; - q++; - } - } - if (dataProcessor != null) { - dataProcessor.applyInline(buf, writePos - readSum, readSum); - } - return readSum >= minLength; - } - - protected int readDatagram(Datagram d, double[] target, int pos) { - int dur = (int) d.getDuration(); - byte[] frameAudio = d.getData(); - assert frameAudio.length / 2 == dur : "expected datagram data length to be " + (dur * 2) + ", found " + frameAudio.length; - for (int i = 0; i < frameAudio.length; i += 2, pos++) { - int sample; - byte lobyte; - byte hibyte; - // big endian: - lobyte = frameAudio[i + 1]; - hibyte = frameAudio[i]; - sample = hibyte << 8 | lobyte & 0xFF; - target[pos] = sample / 32768.0;// normalise to range [-1, 1]; - } - return dur; - } -} diff --git a/marytts-runtime/src/main/java/marytts/unitselection/concat/FdpsolaUnitConcatenator.java b/marytts-runtime/src/main/java/marytts/unitselection/concat/FdpsolaUnitConcatenator.java deleted file mode 100644 index 62c6eb2679..0000000000 --- a/marytts-runtime/src/main/java/marytts/unitselection/concat/FdpsolaUnitConcatenator.java +++ /dev/null @@ -1,601 +0,0 @@ -/** - * Copyright 2007 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.unitselection.concat; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import javax.sound.sampled.AudioInputStream; - -import marytts.modules.phonemiser.Allophone; -import marytts.server.MaryProperties; -import marytts.signalproc.process.FDPSOLAProcessor; -import marytts.unitselection.analysis.Phone; -import marytts.unitselection.select.SelectedUnit; -import marytts.unitselection.select.Target; -import marytts.util.data.Datagram; -import marytts.util.data.audio.DDSAudioInputStream; -import marytts.util.math.MathUtils; - -/** - * A unit concatenator that supports FD-PSOLA based prosody modifications during speech synthesis - * - * @author Oytun Türk, modified by steiner - * - */ -public class FdpsolaUnitConcatenator extends OverlapUnitConcatenator { - - // modification value ranges with hard-coded defaults: - private double minTimeScaleFactor = 0.5; - private double maxTimeScaleFactor = 2.0; - private double minPitchScaleFactor = 0.5; - private double maxPitchScaleFactor = 2.0; - - /** - * - */ - public FdpsolaUnitConcatenator() { - super(); - } - - /** - * Alternative constructor that allows overriding the modification value ranges - * - * @param minTimeScaleFactor - * minimum duration scale factor - * @param maxTimeScaleFactor - * maximum duration scale factor - * @param minPitchScaleFactor - * minimum F0 scale factor - * @param maxPitchScaleFactor - * maximum F0 scale factor - */ - public FdpsolaUnitConcatenator(double minTimeScaleFactor, double maxTimeScaleFactor, double minPitchScaleFactor, - double maxPitchScaleFactor) { - super(); - this.minTimeScaleFactor = minTimeScaleFactor; - this.maxTimeScaleFactor = maxTimeScaleFactor; - this.minPitchScaleFactor = minPitchScaleFactor; - this.maxPitchScaleFactor = maxPitchScaleFactor; - } - - /** - * Get the Datagrams from a List of SelectedUnits as an array of arrays; the number of elements in the array is equal to the - * number of Units, and each element contains that Unit's Datagrams as an array. - * - * @param units - * @return array of Datagram arrays - */ - private Datagram[][] getDatagrams(List units) { - Datagram[][] datagrams = new Datagram[units.size()][]; - for (int i = 0; i < units.size(); i++) { - UnitData unitData = (UnitData) units.get(i).getConcatenationData(); - datagrams[i] = unitData.getFrames(); - } - return datagrams; - } - - /** - * Convenience method to return the rightmost Datagram from each element in a List of SelectedUnits - * - * @param units - * @return rightmost Datagrams as an array - */ - private Datagram[] getRightContexts(List units) { - Datagram[] rightContexts = new Datagram[units.size()]; - for (int i = 0; i < rightContexts.length; i++) { - SelectedUnit unit = units.get(i); - UnitData unitData = (UnitData) unit.getConcatenationData(); - rightContexts[i] = unitData.getRightContextFrame(); - } - return rightContexts; - } - - /** - * Get voicing for every Datagram in a List of SelectedUnits, as an array of arrays of booleans. This queries the phonological - * voicedness value for the Target as defined in the AllophoneSet - * - * @param units - * @return array of boolean voicing arrays - */ - private boolean[][] getVoicings(List units) { - Datagram[][] datagrams = getDatagrams(units); - - boolean[][] voicings = new boolean[datagrams.length][]; - - for (int i = 0; i < datagrams.length; i++) { - Allophone allophone = units.get(i).getTarget().getAllophone(); - - voicings[i] = new boolean[datagrams[i].length]; - - if (allophone != null && allophone.isVoiced()) { - Arrays.fill(voicings[i], true); - } else { - Arrays.fill(voicings[i], false); - } - } - return voicings; - } - - // We can try different things in this function - // 1) Pitch of the selected units can be smoothed without using the target pitch values at all. - // This will involve creating the target f0 values for each frame by ensuing small adjustments and yet reduce pitch - // discontinuity - // 2) Pitch of the selected units can be modified to match the specified target where those target values are smoothed - // 3) A mixture of (1) and (2) can be devised, i.e. to minimize the amount of pitch modification one of the two methods can be - // selected for a given unit - // 4) Pitch segments of selected units can be shifted - // 5) Pitch segments of target units can be shifted - // 6) Pitch slopes can be modified for better matching in concatenation boundaries - private double[][] getPitchScales(List units) { - Datagram[][] datagrams = getDatagrams(units); - int len = datagrams.length; - int i, j; - double averageUnitF0InHz; - double averageTargetF0InHz; - int totalTargetUnits; - double[][] pscales = new double[len][]; - SelectedUnit prevUnit = null; - SelectedUnit unit = null; - SelectedUnit nextUnit = null; - - Target prevTarget = null; - Target target = null; - Target nextTarget = null; - - // Estimation of pitch scale modification amounts - for (i = 0; i < len; i++) { - if (i > 0) - prevUnit = (SelectedUnit) units.get(i - 1); - else - prevUnit = null; - - unit = (SelectedUnit) units.get(i); - - if (i < len - 1) - nextUnit = (SelectedUnit) units.get(i + 1); - else - nextUnit = null; - - // get Targets for these three Units: - if (prevUnit != null) { - prevTarget = prevUnit.getTarget(); - } - target = unit.getTarget(); - if (nextUnit != null) { - nextTarget = nextUnit.getTarget(); - } - - Allophone allophone = unit.getTarget().getAllophone(); - - int totalDatagrams = 0; - averageUnitF0InHz = 0.0; - averageTargetF0InHz = 0.0; - totalTargetUnits = 0; - - // so we are getting the mean F0 for each unit over a 3-unit window?? - // don't process previous Target if it's null or silence: - if (i > 0 && prevTarget != null && !prevTarget.isSilence()) { - for (j = 0; j < datagrams[i - 1].length; j++) { - // why not use voicings? - if (allophone != null && (allophone.isVowel() || allophone.isVoiced())) { - averageUnitF0InHz += ((double) timeline.getSampleRate()) / ((double) datagrams[i - 1][j].getDuration()); - totalDatagrams++; - } - } - - averageTargetF0InHz += prevTarget.getTargetF0InHz(); - totalTargetUnits++; - } - - // don't process Target if it's null or silence: - if (target != null && !target.isSilence()) { - for (j = 0; j < datagrams[i].length; j++) { - if (allophone != null && (allophone.isVowel() || allophone.isVoiced())) { - averageUnitF0InHz += ((double) timeline.getSampleRate()) / ((double) datagrams[i][j].getDuration()); - totalDatagrams++; - } - - averageTargetF0InHz += target.getTargetF0InHz(); - totalTargetUnits++; - } - } - - // don't process next Target if it's null or silence: - if (i < len - 1 && prevTarget != null && !prevTarget.isSilence()) { - for (j = 0; j < datagrams[i + 1].length; j++) { - if (allophone != null && (allophone.isVowel() || allophone.isVoiced())) { - averageUnitF0InHz += ((double) timeline.getSampleRate()) / ((double) datagrams[i + 1][j].getDuration()); - totalDatagrams++; - } - } - - averageTargetF0InHz += nextTarget.getTargetF0InHz(); - totalTargetUnits++; - } - - averageTargetF0InHz /= totalTargetUnits; - averageUnitF0InHz /= totalDatagrams; - // so what was all that for?? these average frequencies are never used... - - pscales[i] = new double[datagrams[i].length]; - - for (j = 0; j < datagrams[i].length; j++) { - if (allophone != null && allophone.isVoiced()) { - /* - * pscales[i][j] = averageTargetF0InHz/averageUnitF0InHz; if (pscales[i][j]>1.2) pscales[i][j]=1.2; if - * (pscales[i][j]<0.8) pscales[i][j]=0.8; - */ - pscales[i][j] = 1.0; - } else { - pscales[i][j] = 1.0; - } - } - } - return pscales; - } - - // We can try different things in this function - // 1) Duration modification factors can be estimated using neighbouring selected and target unit durations - // 2) Duration modification factors can be limited or even set to 1.0 for different phone classes - // 3) Duration modification factors can be limited depending on the previous/next phone class - private double[][] getDurationScales(List units) { - Datagram[][] datagrams = getDatagrams(units); - int len = datagrams.length; - - int i, j; - double[][] tscales = new double[len][]; - int unitDuration; - - double[] unitDurationsInSeconds = new double[datagrams.length]; - - SelectedUnit prevUnit = null; - SelectedUnit unit = null; - SelectedUnit nextUnit = null; - - for (i = 0; i < len; i++) { - unitDuration = 0; - for (j = 0; j < datagrams[i].length; j++) { - if (j == datagrams[i].length - 1) { - // if (rightContexts!=null && rightContexts[i]!=null) - // unitDuration += datagrams[i][j].getDuration();//+rightContexts[i].getDuration(); - // else - unitDuration += datagrams[i][j].getDuration(); - } else - unitDuration += datagrams[i][j].getDuration(); - } - unitDurationsInSeconds[i] = ((double) unitDuration) / timeline.getSampleRate(); - } - - double targetDur, unitDur; - for (i = 0; i < len; i++) { - targetDur = 0.0; - unitDur = 0.0; - // commented out dead code: - // if (false && i>0) - // { - // prevUnit = (SelectedUnit) units.get(i-1); - // targetDur += prevUnit.getTarget().getTargetDurationInSeconds(); - // unitDur += unitDurationsInSeconds[i-1]; - // } - - unit = (SelectedUnit) units.get(i); - targetDur += unit.getTarget().getTargetDurationInSeconds(); - unitDur += unitDurationsInSeconds[i]; - - // commented out dead code: - // if (false && i1.2) - // tscales[i][j]=1.2; - // if (tscales[i][j]<0.8) - // tscales[i][j]=0.8; - - // tscales[i][j] = 1.2; - } - logger.debug("time scaling factor for unit " + unit.getTarget().getName() + " -> " + targetDur / unitDur); - } - return tscales; - } - - // private double[][] getSyllableBasedPitchScales(List units) { - // List phones = ProsodyAnalyzer.parseIntoPhones(units, timeline.getSampleRate()); - // List syllables = Syllable.parseIntoSyllables(phones); - // ListIterator syllableIterator = syllables.listIterator(); - // while (syllableIterator.hasNext()) { - // if (!syllableIterator.hasPrevious()) { - // continue; - // } - // // TODO unfinished! - // } - // return null; - // } - - private double[][] getPhoneBasedDurationScales(List units) { - - List timeScaleFactors = prosodyAnalyzer.getDurationFactors(); - - // finally, initialize the tscales array... - double[][] tscales = new double[timeScaleFactors.size()][]; - Datagram[][] datagrams = getDatagrams(units); - for (int i = 0; i < tscales.length; i++) { - tscales[i] = new double[datagrams[i].length]; - // ...which currently provides the same time scale factor for every datagram in a selected unit: - Arrays.fill(tscales[i], timeScaleFactors.get(i)); - } - - // for quick and dirty debugging, dump tscales to Praat DurationTier: - try { - prosodyAnalyzer.writePraatDurationTier(MaryProperties.maryBase() + "/tscales.DurationTier"); - } catch (IOException e) { - logger.warn("Could not dump tscales to file"); - } - - return tscales; - } - - /** - * Convenience method to grep those SelectedUnits from a List which have positive duration - * - * @param units - * @return units with positive duration - */ - @Deprecated - private List getNonEmptyUnits(List units) { - ArrayList nonEmptyUnits = new ArrayList(units.size()); - for (SelectedUnit unit : units) { - UnitData unitData = (UnitData) unit.getConcatenationData(); - if (unitData.getUnitDuration() > 0 && unit.getTarget().getMaryxmlElement() != null) { - nonEmptyUnits.add(unit); - } - } - return nonEmptyUnits; - } - - protected Datagram[][] getRealizedDatagrams(List phones) { - List datagramList = new ArrayList(); - for (Phone phone : phones) { - if (phone.getLeftTargetDuration() > 0) { - Datagram[] leftDatagrams = phone.getLeftUnitFrames(); - datagramList.add(leftDatagrams); - } - if (phone.getRightTargetDuration() > 0) { - Datagram[] rightDatagrams = phone.getRightUnitFrames(); - datagramList.add(rightDatagrams); - } - } - Datagram[][] datagramArray = datagramList.toArray(new Datagram[datagramList.size()][]); - return datagramArray; - } - - protected Datagram[] getRealizedRightContexts(List phones) { - List datagramList = new ArrayList(); - for (Phone phone : phones) { - if (phone.getLeftTargetDuration() > 0) { - UnitData leftUnitData = phone.getLeftUnitData(); - Datagram leftRightContext = leftUnitData.getRightContextFrame(); - datagramList.add(leftRightContext); - } - if (phone.getRightTargetDuration() > 0) { - UnitData rightUnitData = phone.getRightUnitData(); - Datagram rightRightContext = rightUnitData.getRightContextFrame(); - datagramList.add(rightRightContext); - } - } - Datagram[] datagramArray = datagramList.toArray(new Datagram[datagramList.size()]); - return datagramArray; - } - - private boolean[][] getRealizedVoicings(List phones) { - List voicingList = new ArrayList(); - for (Phone phone : phones) { - boolean voiced = phone.isVoiced(); - if (phone.getLeftTargetDuration() > 0) { - int leftNumberOfFrames = phone.getNumberOfLeftUnitFrames(); - boolean[] leftVoiceds = new boolean[leftNumberOfFrames]; - Arrays.fill(leftVoiceds, voiced); - voicingList.add(leftVoiceds); - } - if (phone.getRightTargetDuration() > 0) { - int rightNumberOfFrames = phone.getNumberOfRightUnitFrames(); - boolean[] rightVoiceds = new boolean[rightNumberOfFrames]; - Arrays.fill(rightVoiceds, voiced); - voicingList.add(rightVoiceds); - } - } - boolean[][] voicingArray = voicingList.toArray(new boolean[voicingList.size()][]); - return voicingArray; - } - - private double[][] getRealizedTimeScales(List phones) { - List durationFactorList = new ArrayList(phones.size()); - for (Phone phone : phones) { - if (phone.getLeftTargetDuration() > 0) { - int leftNumberOfFrames = phone.getNumberOfLeftUnitFrames(); - double leftDurationFactor = phone.getLeftDurationFactor(); - // scale the factor to reasonably safe values: - if (leftDurationFactor < minTimeScaleFactor) { - String message = "Left duration factor (" + leftDurationFactor + ") for phone " + phone + " too small;"; - leftDurationFactor = minTimeScaleFactor; - message += " clipped to " + leftDurationFactor; - logger.debug(message); - } else if (leftDurationFactor > maxTimeScaleFactor) { - String message = "Left duration factor (" + leftDurationFactor + ") for phone " + phone + " too large;"; - leftDurationFactor = maxTimeScaleFactor; - message += " clipped to " + leftDurationFactor; - logger.debug(message); - } - double[] leftDurationFactors = new double[leftNumberOfFrames]; - Arrays.fill(leftDurationFactors, leftDurationFactor); - durationFactorList.add(leftDurationFactors); - } - if (phone.getRightTargetDuration() > 0) { - int rightNumberOfFrames = phone.getNumberOfRightUnitFrames(); - double rightDurationFactor = phone.getRightDurationFactor(); - if (phone.isTransient()) { - rightDurationFactor = 1; // never modify the duration of a burst - } - // scale the factor to reasonably safe values: - if (rightDurationFactor < minTimeScaleFactor) { - String message = "Right duration factor (" + rightDurationFactor + ") for phone " + phone + " too small;"; - rightDurationFactor = minTimeScaleFactor; - message += " clipped to " + rightDurationFactor; - logger.debug(message); - } else if (rightDurationFactor > maxTimeScaleFactor) { - String message = "Right duration factor (" + rightDurationFactor + ") for phone " + phone + " too large;"; - rightDurationFactor = maxTimeScaleFactor; - message += " clipped to " + rightDurationFactor; - logger.debug(message); - } - double[] rightDurationFactors = new double[rightNumberOfFrames]; - Arrays.fill(rightDurationFactors, rightDurationFactor); - durationFactorList.add(rightDurationFactors); - } - } - double[][] durationFactorArray = durationFactorList.toArray(new double[durationFactorList.size()][]); - return durationFactorArray; - } - - private double[][] getRealizedPitchScales(List phones) { - List f0FactorList = new ArrayList(phones.size()); - for (Phone phone : phones) { - if (phone.getLeftTargetDuration() > 0) { - int leftNumberOfFrames = phone.getNumberOfLeftUnitFrames(); - double[] leftF0Factors = phone.getLeftF0Factors(); - boolean clipped = MathUtils.clipRange(leftF0Factors, minPitchScaleFactor, maxPitchScaleFactor); - if (clipped) { - logger.debug("Left F0 factors for phone " + phone + " contained out-of-range values; clipped to [" - + minPitchScaleFactor + ", " + maxPitchScaleFactor + "]"); - } - f0FactorList.add(leftF0Factors); - } - if (phone.getRightTargetDuration() > 0) { - int rightNumberOfFrames = phone.getNumberOfRightUnitFrames(); - double[] rightF0Factors = phone.getRightF0Factors(); - boolean clipped = MathUtils.clipRange(rightF0Factors, minPitchScaleFactor, maxPitchScaleFactor); - if (clipped) { - logger.debug("Left F0 factors for phone " + phone + " contained out-of-range values; clipped to [" - + minPitchScaleFactor + ", " + maxPitchScaleFactor + "]"); - } - f0FactorList.add(rightF0Factors); - } - } - double[][] f0FactorArray = f0FactorList.toArray(new double[f0FactorList.size()][]); - return f0FactorArray; - } - - /** - * Generate audio to match the target pitchmarks as closely as possible. - * - * @param units - * @return - * @throws IOException - */ - protected AudioInputStream generateAudioStream(List units) throws IOException { - // gather arguments for FDPSOLA processing: - // Datagram[][] datagrams = getDatagrams(units); - // Datagram[] rightContexts = getRightContexts(units); - // boolean[][] voicings = getVoicings(units); - // double[][] pscales = getPitchScales(units); - // double[][] tscales = getDurationScales(units); - // double[][] tscales = getPhoneBasedDurationScales(units); - - List realizedPhones = prosodyAnalyzer.getRealizedPhones(); - Datagram[][] datagrams = getRealizedDatagrams(realizedPhones); - Datagram[] rightContexts = getRealizedRightContexts(realizedPhones); - boolean[][] voicings = getRealizedVoicings(realizedPhones); - double[][] tscales = getRealizedTimeScales(realizedPhones); - double[][] pscales = getRealizedPitchScales(realizedPhones); - - // process into audio stream: - DDSAudioInputStream stream = (new FDPSOLAProcessor()).processDecrufted(datagrams, rightContexts, audioformat, voicings, - pscales, tscales); - - // update durations from processed Datagrams: - // updateUnitDataDurations(units, datagrams); - updateRealizedUnitDataDurations(realizedPhones, datagrams); - - return stream; - } - - /** - * Explicitly propagate durations of Datagrams to UnitData for each SelectedUnit; those durations are otherwise oblivious to - * the data they describe... - * - * @param units - * whose data should have its durations updated - * @param datagrams - * processed array of arrays of Datagrams which had their durations updated in - * {@link FDPSOLAProcessor#processDecrufted} - */ - private void updateUnitDataDurations(List units, Datagram[][] datagrams) { - for (int i = 0; i < datagrams.length; i++) { - SelectedUnit unit = units.get(i); - UnitData unitData = (UnitData) unit.getConcatenationData(); - int unitDuration = 0; - for (int j = 0; j < datagrams[i].length; j++) { - int datagramDuration = (int) datagrams[i][j].getDuration(); - unitData.getFrame(j).setDuration(datagramDuration); - unitDuration += datagramDuration; - } - unitData.setUnitDuration(unitDuration); - } - } - - private void updateRealizedUnitDataDurations(List phones, Datagram[][] datagrams) { - int phIndex = 0; - for (Phone phone : phones) { - if (phone.getLeftTargetDuration() > 0) { - UnitData leftUnitData = phone.getLeftUnitData(); - int leftUnitDataDuration = 0; - for (int dg = 0; dg < datagrams[phIndex].length; dg++) { - int datagramDuration = (int) datagrams[phIndex][dg].getDuration(); - leftUnitData.getFrame(dg).setDuration(datagramDuration); - leftUnitDataDuration += datagramDuration; - } - phIndex++; - leftUnitData.setUnitDuration(leftUnitDataDuration); - } - if (phone.getRightTargetDuration() > 0) { - UnitData rightUnitData = phone.getRightUnitData(); - int rightUnitDataDuration = 0; - for (int dg = 0; dg < datagrams[phIndex].length; dg++) { - int datagramDuration = (int) datagrams[phIndex][dg].getDuration(); - rightUnitData.getFrame(dg).setDuration(datagramDuration); - rightUnitDataDuration += datagramDuration; - } - phIndex++; - rightUnitData.setUnitDuration(rightUnitDataDuration); - } - } - } -} diff --git a/marytts-runtime/src/main/java/marytts/unitselection/concat/HnmUnitConcatenator.java b/marytts-runtime/src/main/java/marytts/unitselection/concat/HnmUnitConcatenator.java deleted file mode 100644 index fe4d78363a..0000000000 --- a/marytts-runtime/src/main/java/marytts/unitselection/concat/HnmUnitConcatenator.java +++ /dev/null @@ -1,251 +0,0 @@ -/** - * Copyright 2007 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * Permission is hereby granted, free of charge, to use and distribute - * this software and its documentation without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of this work, and to - * permit persons to whom this work is furnished to do so, subject to - * the following conditions: - * - * 1. The code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * 2. Any modifications must be clearly marked as such. - * 3. Original authors' names are not deleted. - * 4. The authors' names are not used to endorse or promote products - * derived from this software without specific prior written - * permission. - * - * DFKI GMBH AND THE CONTRIBUTORS TO THIS WORK DISCLAIM ALL WARRANTIES WITH - * REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL DFKI GMBH NOR THE - * CONTRIBUTORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL - * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR - * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS - * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF - * THIS SOFTWARE. - */ - -package marytts.unitselection.concat; - -import java.io.IOException; -import java.util.List; - -import javax.sound.sampled.AudioInputStream; - -import marytts.signalproc.adaptation.prosody.BasicProsodyModifierParams; -import marytts.signalproc.sinusoidal.hntm.analysis.HntmAnalyzerParams; -import marytts.signalproc.sinusoidal.hntm.analysis.HntmSpeechFrame; -import marytts.signalproc.sinusoidal.hntm.analysis.HntmSpeechSignal; -import marytts.signalproc.sinusoidal.hntm.synthesis.HntmSynthesizedSignal; -import marytts.signalproc.sinusoidal.hntm.synthesis.HntmSynthesizer; -import marytts.signalproc.sinusoidal.hntm.synthesis.HntmSynthesizerParams; -import marytts.unitselection.data.HnmDatagram; -import marytts.unitselection.data.Unit; -import marytts.unitselection.select.SelectedUnit; -import marytts.util.data.BufferedDoubleDataSource; -import marytts.util.data.Datagram; -import marytts.util.data.audio.DDSAudioInputStream; -import marytts.util.math.MathUtils; - -/** - * A unit concatenator for harmonics plus noise based speech synthesis - * - * @author Oytun Türk - * - */ -public class HnmUnitConcatenator extends OverlapUnitConcatenator { - - public HnmUnitConcatenator() { - super(); - } - - /** - * Get the raw audio material for each unit from the timeline. - * - * @param units - */ - protected void getDatagramsFromTimeline(List units) throws IOException { - for (SelectedUnit unit : units) { - assert !unit.getUnit().isEdgeUnit() : "We should never have selected any edge units!"; - HnmUnitData unitData = new HnmUnitData(); - unit.setConcatenationData(unitData); - int nSamples = 0; - int unitSize = unitToTimeline(unit.getUnit().duration); // convert to timeline samples - long unitStart = unitToTimeline(unit.getUnit().startTime); // convert to timeline samples - // System.out.println("Unit size "+unitSize+", pitchmarksInUnit "+pitchmarksInUnit); - // System.out.println(unitStart/((float)timeline.getSampleRate())); - // System.out.println("Unit index = " + unit.getUnit().getIndex()); - - Datagram[] datagrams = timeline.getDatagrams(unitStart, (long) unitSize); - unitData.setFrames(datagrams); - - // one left context period for windowing: - Datagram leftContextFrame = null; - Unit prevInDB = database.getUnitFileReader().getPreviousUnit(unit.getUnit()); - long unitPrevStart = unitToTimeline(prevInDB.startTime); // convert to timeline samples - if (prevInDB != null && !prevInDB.isEdgeUnit()) { - long unitPrevSize = unitToTimeline(prevInDB.duration); - Datagram[] unitPrevDatagrams = timeline.getDatagrams(unitPrevStart, (long) unitPrevSize); - // leftContextFrame = timeline.getDatagram(unitPrevStart); - if (unitPrevDatagrams != null && unitPrevDatagrams.length > 0) { - leftContextFrame = unitPrevDatagrams[unitPrevDatagrams.length - 1]; - } - unitData.setLeftContextFrame(leftContextFrame); - } - - // one right context period for windowing: - Datagram rightContextFrame = null; - Unit nextInDB = database.getUnitFileReader().getNextUnit(unit.getUnit()); - if (nextInDB != null && !nextInDB.isEdgeUnit()) { - rightContextFrame = timeline.getDatagram(unitStart + unitSize); - unitData.setRightContextFrame(rightContextFrame); - } - } - } - - /** - * Generate audio to match the target pitchmarks as closely as possible. - * - * @param units - * @return - */ - protected AudioInputStream generateAudioStream(List units) { - int len = units.size(); - Datagram[][] datagrams = new Datagram[len][]; - Datagram[] leftContexts = new Datagram[len]; - Datagram[] rightContexts = new Datagram[len]; - for (int i = 0; i < len; i++) { - SelectedUnit unit = units.get(i); - HnmUnitData unitData = (HnmUnitData) unit.getConcatenationData(); - assert unitData != null : "Should not have null unitdata here"; - Datagram[] frames = unitData.getFrames(); - assert frames != null : "Cannot generate audio from null frames"; - // Generate audio from frames - datagrams[i] = frames; - - Unit prevInDB = database.getUnitFileReader().getPreviousUnit(unit.getUnit()); - Unit prevSelected; - if (i == 0) - prevSelected = null; - else - prevSelected = units.get(i - 1).getUnit(); - if (prevInDB != null && !prevInDB.equals(prevSelected)) { - // Only use left context if we have a previous unit in the DB is not the - // same as the previous selected unit. - leftContexts[i] = (HnmDatagram) unitData.getLeftContextFrame(); // may be null - } - - Unit nextInDB = database.getUnitFileReader().getNextUnit(unit.getUnit()); - Unit nextSelected; - if (i + 1 == len) - nextSelected = null; - else - nextSelected = units.get(i + 1).getUnit(); - if (nextInDB != null && !nextInDB.equals(nextSelected)) { - // Only use right context if we have a next unit in the DB is not the - // same as the next selected unit. - rightContexts[i] = unitData.getRightContextFrame(); // may be null - } - } - - BufferedDoubleDataSource audioSource = synthesize(datagrams, leftContexts, rightContexts); - return new DDSAudioInputStream(audioSource, audioformat); - } - - protected BufferedDoubleDataSource synthesize(Datagram[][] datagrams, Datagram[] leftContexts, Datagram[] rightContexts) { - HntmSynthesizer s = new HntmSynthesizer(); - // TO DO: These should come from timeline and user choices... - HntmAnalyzerParams analysisParams = new HntmAnalyzerParams(); - HntmSynthesizerParams synthesisParams = new HntmSynthesizerParams(); - BasicProsodyModifierParams pmodParams = new BasicProsodyModifierParams(); - int samplingRateInHz = 16000; - - int totalFrm = 0; - int i, j; - float originalDurationInSeconds = 0.0f; - float deltaTimeInSeconds; - long length; - - for (i = 0; i < datagrams.length; i++) { - for (j = 0; j < datagrams[i].length; j++) { - if (datagrams[i][j] != null && (datagrams[i][j] instanceof HnmDatagram)) { - totalFrm++; - length = ((HnmDatagram) datagrams[i][j]).getDuration(); - // deltaTimeInSeconds = SignalProcUtils.sample2time(length, samplingRateInHz); - deltaTimeInSeconds = ((HntmSpeechFrame) ((HnmDatagram) datagrams[i][j]).getFrame()).deltaAnalysisTimeInSeconds; - originalDurationInSeconds += deltaTimeInSeconds; - - // System.out.println("Unit duration = " + String.valueOf(length)); - } - } - } - - HntmSpeechSignal hnmSignal = null; - hnmSignal = new HntmSpeechSignal(totalFrm, samplingRateInHz, originalDurationInSeconds); - HntmSpeechFrame[] leftContextFrames = new HntmSpeechFrame[totalFrm]; - HntmSpeechFrame[] rightContextFrames = new HntmSpeechFrame[totalFrm]; - // - - int frameCount = 0; - float tAnalysisInSeconds = 0.0f; - for (i = 0; i < datagrams.length; i++) { - for (j = 0; j < datagrams[i].length; j++) { - if (datagrams[i][j] != null && (datagrams[i][j] instanceof HnmDatagram) && frameCount < totalFrm) { - tAnalysisInSeconds += ((HntmSpeechFrame) ((HnmDatagram) datagrams[i][j]).getFrame()).deltaAnalysisTimeInSeconds; - - hnmSignal.frames[frameCount] = ((HnmDatagram) datagrams[i][j]).getFrame(); - hnmSignal.frames[frameCount].tAnalysisInSeconds = tAnalysisInSeconds; - - // tAnalysisInSeconds += SignalProcUtils.sample2time(((HnmDatagram)datagrams[i][j]).getDuration(), - // samplingRateInHz); - - if (j == 0) { - if (leftContexts[i] != null && (leftContexts[i] instanceof HnmDatagram)) - leftContextFrames[frameCount] = ((HnmDatagram) leftContexts[i]).getFrame(); - } else { - if (datagrams[i][j - 1] != null && (datagrams[i][j - 1] instanceof HnmDatagram)) - leftContextFrames[frameCount] = ((HnmDatagram) datagrams[i][j - 1]).getFrame(); - } - - if (j == datagrams[i].length - 1) { - if (rightContexts[i] != null && (rightContexts[i] instanceof HnmDatagram)) - rightContextFrames[frameCount] = ((HnmDatagram) rightContexts[i]).getFrame(); - } else { - if (datagrams[i][j + 1] != null && (datagrams[i][j + 1] instanceof HnmDatagram)) - rightContextFrames[frameCount] = ((HnmDatagram) datagrams[i][j + 1]).getFrame(); - } - - frameCount++; - } - } - } - - HntmSynthesizedSignal ss = null; - if (totalFrm > 0) { - ss = s.synthesize(hnmSignal, leftContextFrames, rightContextFrames, pmodParams, null, analysisParams, synthesisParams); - // FileUtils.writeTextFile(hnmSignal.getAnalysisTimes(), "d:\\hnmAnalysisTimes1.txt"); - // FileUtils.writeTextFile(ss.output, "d:\\output.txt"); - if (ss.output != null) - ss.output = MathUtils.multiply(ss.output, 1.0 / 32768.0); - } - - if (ss != null && ss.output != null) - return new BufferedDoubleDataSource(ss.output); - else - return null; - } - - public static class HnmUnitData extends OverlapUnitConcatenator.OverlapUnitData { - protected Datagram leftContextFrame; - - public void setLeftContextFrame(Datagram aLeftContextFrame) { - this.leftContextFrame = aLeftContextFrame; - } - - public Datagram getLeftContextFrame() { - return leftContextFrame; - } - } -} diff --git a/marytts-runtime/src/main/java/marytts/unitselection/concat/OverlapUnitConcatenator.java b/marytts-runtime/src/main/java/marytts/unitselection/concat/OverlapUnitConcatenator.java deleted file mode 100644 index 67022d9259..0000000000 --- a/marytts-runtime/src/main/java/marytts/unitselection/concat/OverlapUnitConcatenator.java +++ /dev/null @@ -1,206 +0,0 @@ -/** - * Copyright 2000-2009 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.unitselection.concat; - -import java.io.IOException; -import java.util.List; - -import javax.sound.sampled.AudioInputStream; - -import marytts.unitselection.data.Unit; -import marytts.unitselection.select.SelectedUnit; -import marytts.util.data.BufferedDoubleDataSource; -import marytts.util.data.Datagram; -import marytts.util.data.DoubleDataSource; -import marytts.util.data.audio.DDSAudioInputStream; - -public class OverlapUnitConcatenator extends BaseUnitConcatenator { - - public OverlapUnitConcatenator() { - super(); - } - - /** - * Get the raw audio material for each unit from the timeline. - * - * @param units - */ - protected void getDatagramsFromTimeline(List units) throws IOException { - for (SelectedUnit unit : units) { - assert !unit.getUnit().isEdgeUnit() : "We should never have selected any edge units!"; - OverlapUnitData unitData = new OverlapUnitData(); - unit.setConcatenationData(unitData); - int nSamples = 0; - int unitSize = unitToTimeline(unit.getUnit().duration); // convert to timeline samples - long unitStart = unitToTimeline(unit.getUnit().startTime); // convert to timeline samples - // System.out.println("Unit size "+unitSize+", pitchmarksInUnit "+pitchmarksInUnit); - // System.out.println(unitStart/((float)timeline.getSampleRate())); - // System.out.println("Unit index = " + unit.getUnit().getIndex()); - - Datagram[] datagrams = timeline.getDatagrams(unitStart, (long) unitSize); - unitData.setFrames(datagrams); - // one right context period for windowing: - Datagram rightContextFrame = null; - Unit nextInDB = database.getUnitFileReader().getNextUnit(unit.getUnit()); - if (nextInDB != null && !nextInDB.isEdgeUnit()) { - rightContextFrame = timeline.getDatagram(unitStart + unitSize); - unitData.setRightContextFrame(rightContextFrame); - } - } - } - - /** - * Determine target pitchmarks (= duration and f0) for each unit. - * - * @param units - */ - protected void determineTargetPitchmarks(List units) { - for (SelectedUnit unit : units) { - UnitData unitData = (UnitData) unit.getConcatenationData(); - assert unitData != null : "Should not have null unitdata here"; - Datagram[] datagrams = unitData.getFrames(); - Datagram[] frames = null; // frames to realise - // The number and duration of the frames to realise - // must be the result of the target pitchmark computation. - - // Set target pitchmarks, - // either by copying from units (data-driven) - // or by computing from target (model-driven) - int unitDuration = 0; - int nZeroLengthDatagrams = 0; - for (int i = 0; i < datagrams.length; i++) { - int dur = (int) datagrams[i].getDuration(); - if (dur == 0) - nZeroLengthDatagrams++; - unitDuration += datagrams[i].getDuration(); - } - if (nZeroLengthDatagrams > 0) { - logger.warn("Unit " + unit + " contains " + nZeroLengthDatagrams + " zero-length datagrams -- removing them"); - Datagram[] dummy = new Datagram[datagrams.length - nZeroLengthDatagrams]; - for (int i = 0, j = 0; i < datagrams.length; i++) { - if (datagrams[i].getDuration() > 0) { - dummy[j++] = datagrams[i]; - } - } - datagrams = dummy; - unitData.setFrames(datagrams); - } - if (unit.getTarget().isSilence()) { - int targetDuration = Math.round(unit.getTarget().getTargetDurationInSeconds() * audioformat.getSampleRate()); - if (targetDuration > 0 && datagrams != null && datagrams.length > 0) { - int firstPeriodDur = (int) datagrams[0].getDuration(); - if (targetDuration < firstPeriodDur) { - logger.debug("For " + unit + ", adjusting target duration to be at least one period: " - + (firstPeriodDur / audioformat.getSampleRate()) + " s instead of requested " - + unit.getTarget().getTargetDurationInSeconds() + " s"); - targetDuration = firstPeriodDur; - } - if (unitDuration < targetDuration) { - // insert silence in the middle - frames = new Datagram[datagrams.length + 1]; - int mid = (datagrams.length + 1) / 2; - System.arraycopy(datagrams, 0, frames, 0, mid); - if (mid < datagrams.length) { - System.arraycopy(datagrams, mid, frames, mid + 1, datagrams.length - mid); - } - frames[mid] = createZeroDatagram(targetDuration - unitDuration); - } else { // unitDuration >= targetDuration - // cut frames from the middle - int midright = (datagrams.length + 1) / 2; // first frame of the right part - int midleft = midright - 1; // last frame of the left part - while (unitDuration > targetDuration && midright < datagrams.length) { - unitDuration -= datagrams[midright].getDuration(); - midright++; - if (unitDuration > targetDuration && midleft > 0) { // force it to leave at least one frame, therefore - // > 0 - unitDuration -= datagrams[midleft].getDuration(); - midleft--; - } - } - frames = new Datagram[midleft + 1 + datagrams.length - midright]; - assert midleft >= 0; - System.arraycopy(datagrams, 0, frames, 0, midleft + 1); - if (midright < datagrams.length) { - System.arraycopy(datagrams, midright, frames, midleft + 1, datagrams.length - midright); - } - } - unitDuration = targetDuration; // now they are the same - } else { // unitSize == 0, we have a zero-length silence unit - // artificial silence data: - frames = new Datagram[] { createZeroDatagram(targetDuration) }; - unitDuration = targetDuration; - } - } else { // not silence - // take unit as is - frames = datagrams; - } - unitData.setUnitDuration(unitDuration); - unitData.setFrames(frames); - } - } - - /** - * Generate audio to match the target pitchmarks as closely as possible. - * - * @param units - * @return - * @throws IOException - */ - protected AudioInputStream generateAudioStream(List units) throws IOException { - int len = units.size(); - Datagram[][] datagrams = new Datagram[len][]; - Datagram[] rightContexts = new Datagram[len]; - for (int i = 0; i < len; i++) { - SelectedUnit unit = units.get(i); - OverlapUnitData unitData = (OverlapUnitData) unit.getConcatenationData(); - assert unitData != null : "Should not have null unitdata here"; - Datagram[] frames = unitData.getFrames(); - assert frames != null : "Cannot generate audio from null frames"; - // Generate audio from frames - datagrams[i] = frames; - Unit nextInDB = database.getUnitFileReader().getNextUnit(unit.getUnit()); - Unit nextSelected; - if (i + 1 == len) - nextSelected = null; - else - nextSelected = units.get(i + 1).getUnit(); - if (nextInDB != null && !nextInDB.equals(nextSelected)) { - // Only use right context if we have a next unit in the DB is not the - // same as the next selected unit. - rightContexts[i] = unitData.getRightContextFrame(); // may be null - } - } - - DoubleDataSource audioSource = new DatagramOverlapDoubleDataSource(datagrams, rightContexts); - return new DDSAudioInputStream(new BufferedDoubleDataSource(audioSource), audioformat); - } - - public static class OverlapUnitData extends BaseUnitConcatenator.UnitData { - protected Datagram rightContextFrame; - - public void setRightContextFrame(Datagram aRightContextFrame) { - this.rightContextFrame = aRightContextFrame; - } - - public Datagram getRightContextFrame() { - return rightContextFrame; - } - } -} diff --git a/marytts-runtime/src/main/java/marytts/unitselection/concat/UnitConcatenator.java b/marytts-runtime/src/main/java/marytts/unitselection/concat/UnitConcatenator.java deleted file mode 100644 index 5232ba8871..0000000000 --- a/marytts-runtime/src/main/java/marytts/unitselection/concat/UnitConcatenator.java +++ /dev/null @@ -1,61 +0,0 @@ -/** - * Copyright 2006 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.unitselection.concat; - -import java.io.IOException; -import java.util.List; - -import javax.sound.sampled.AudioFormat; -import javax.sound.sampled.AudioInputStream; - -import marytts.unitselection.data.UnitDatabase; -import marytts.unitselection.select.SelectedUnit; - -/** - * Concatenates the units of an utterance and returns an audio stream - * - * @author Marc Schröder, Anna Hunecke - * - */ -public interface UnitConcatenator { - /** - * Initialise the unit concatenator from the database. - * - * @param database - */ - public void load(UnitDatabase database); - - /** - * Build the audio stream from the units - * - * @param units - * the units - * @return the resulting audio stream - */ - public AudioInputStream getAudio(List units) throws IOException; - - /** - * Provide the audio format which will be produced by this unit concatenator. - * - * @return the audio format - */ - public AudioFormat getAudioFormat(); - -} diff --git a/marytts-runtime/src/main/java/marytts/unitselection/data/DiphoneUnit.java b/marytts-runtime/src/main/java/marytts/unitselection/data/DiphoneUnit.java deleted file mode 100644 index bc57790ec9..0000000000 --- a/marytts-runtime/src/main/java/marytts/unitselection/data/DiphoneUnit.java +++ /dev/null @@ -1,65 +0,0 @@ -/** - * Copyright 2006 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.unitselection.data; - -public class DiphoneUnit extends Unit { - public final Unit left; - public final Unit right; - - public DiphoneUnit(Unit left, Unit right) { - super(left.startTime, left.duration + right.duration, left.index); - this.left = left; - this.right = right; - } - - public int getIndex() { - throw new IllegalStateException("This method should not be called for DiphoneUnits."); - } - - public boolean isEdgeUnit() { - throw new IllegalStateException("This method should not be called for DiphoneUnits."); - } - - public String toString() { - return "diphoneunit " + index + " start: " + startTime + ", duration: " + duration; - } - - /** - * inspired by http://www.artima.com/lejava/articles/equality.html - */ - @Override - public boolean equals(Object other) { - boolean result = false; - if (other instanceof DiphoneUnit) { - DiphoneUnit that = (DiphoneUnit) other; - result = (this.left.equals(that.left) && this.right.equals(that.right)); - } - return result; - } - - /** - * inspired by http://www.artima.com/lejava/articles/equality.html - */ - @Override - public int hashCode() { - return (41 * (41 + this.left.hashCode()) + this.right.hashCode()); - } - -} diff --git a/marytts-runtime/src/main/java/marytts/unitselection/data/DiphoneUnitDatabase.java b/marytts-runtime/src/main/java/marytts/unitselection/data/DiphoneUnitDatabase.java deleted file mode 100644 index cb7b37f3cb..0000000000 --- a/marytts-runtime/src/main/java/marytts/unitselection/data/DiphoneUnitDatabase.java +++ /dev/null @@ -1,159 +0,0 @@ -/** - * Copyright 2006 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.unitselection.data; - -import gnu.trove.TIntHashSet; - -import java.util.ArrayList; -import java.util.List; - -import marytts.features.FeatureVector; -import marytts.unitselection.select.DiphoneTarget; -import marytts.unitselection.select.HalfPhoneTarget; -import marytts.unitselection.select.Target; -import marytts.unitselection.select.viterbi.ViterbiCandidate; -import marytts.util.MaryUtils; -import marytts.util.dom.DomUtils; - -import org.w3c.dom.Element; - -public class DiphoneUnitDatabase extends UnitDatabase { - - public DiphoneUnitDatabase() { - super(); - logger = MaryUtils.getLogger("DiphoneUnitDatabase"); - } - - /** - * Preselect a set of candidates that could be used to realise the given target. - * - * @param target - * a Target object representing an optimal unit - * @return an unsorted ArrayList of ViterbiCandidates, each containing the (same) target and a - * (different) Unit object - */ - @Override - public List getCandidates(Target target) { - if (!(target instanceof DiphoneTarget)) - return super.getCandidates(target); - // Basic idea: get the candidates for each half phone separately, - // but retain only those that are part of a suitable diphone - DiphoneTarget diphoneTarget = (DiphoneTarget) target; - HalfPhoneTarget left = diphoneTarget.left; - HalfPhoneTarget right = diphoneTarget.right; - - // BEGIN blacklisting - // The point of this is to get the value of the "blacklist" attribute in the first child element of the MaryXML - // and store it in the blacklist String variable. - // This code seems rather inelegant; perhaps there is a better way to access the MaryXML from this method? - String blacklist = ""; - String unitBasename = "This must never be null or the empty string!"; // otherwise candidate selection fails! - Element targetElement = left.getMaryxmlElement(); - if (targetElement == null) { - targetElement = right.getMaryxmlElement(); - } - blacklist = DomUtils.getAttributeFromClosestAncestorOfAnyKind(targetElement, "blacklist"); - // END blacklisting - - // TODO shouldn't leftName and rightName just call appropriate methods of DiphoneTarget? - String leftName = left.getName().substring(0, left.getName().lastIndexOf("_")); - String rightName = right.getName().substring(0, right.getName().lastIndexOf("_")); - int iPhoneme = targetCostFunction.getFeatureDefinition().getFeatureIndex("phone"); - byte bleftName = targetCostFunction.getFeatureDefinition().getFeatureValueAsByte(iPhoneme, leftName); - byte brightName = targetCostFunction.getFeatureDefinition().getFeatureValueAsByte(iPhoneme, rightName); - FeatureVector[] fvs = targetCostFunction.getFeatureVectors(); - - // HashSet candidateUnitSet = new HashSet(); - TIntHashSet candidateUnitSet = new TIntHashSet(); - - // Pre-select candidates for the left half, but retain only - // those that belong to appropriate diphones: - int[] clist = (int[]) preselectionCART.interpret(left, backtrace); - logger.debug("For target " + target + ", selected " + clist.length + " units"); - - // Now, clist is an array of halfphone unit indexes. - for (int i = 0; i < clist.length; i++) { - Unit unit = unitReader.units[clist[i]]; - FeatureVector fv = fvs != null ? fvs[unit.index] : targetCostFunction.getFeatureVector(unit); - byte bunitName = fv.byteValuedDiscreteFeatures[iPhoneme]; - // force correct phone symbol: - if (bunitName != bleftName) - continue; - int iRightNeighbour = clist[i] + 1; - if (iRightNeighbour < numUnits) { - Unit rightNeighbour = unitReader.units[iRightNeighbour]; - FeatureVector rfv = fvs != null ? fvs[iRightNeighbour] : targetCostFunction.getFeatureVector(rightNeighbour); - byte brightUnitName = rfv.byteValuedDiscreteFeatures[iPhoneme]; - if (brightUnitName == brightName) { - // Found a diphone -- add it to candidates - // DiphoneUnit diphoneUnit = new DiphoneUnit(unit, rightNeighbour); - // candidateUnitSet.add(diphoneUnit); - candidateUnitSet.add(unit.index); - } - } - } - // Pre-select candidates for the right half, but retain only - // those that belong to appropriate diphones: - clist = (int[]) preselectionCART.interpret(right, backtrace); - logger.debug("For target " + target + ", selected " + clist.length + " units"); - - // Now, clist is an array of halfphone unit indexes. - for (int i = 0; i < clist.length; i++) { - Unit unit = unitReader.units[clist[i]]; - FeatureVector fv = fvs != null ? fvs[unit.index] : targetCostFunction.getFeatureVector(unit); - byte bunitName = fv.byteValuedDiscreteFeatures[iPhoneme]; - // force correct phone symbol: - if (bunitName != brightName) - continue; - int iLeftNeighbour = clist[i] - 1; - if (iLeftNeighbour >= 0) { - Unit leftNeighbour = unitReader.units[iLeftNeighbour]; - FeatureVector lfv = fvs != null ? fvs[iLeftNeighbour] : targetCostFunction.getFeatureVector(leftNeighbour); - byte bleftUnitName = lfv.byteValuedDiscreteFeatures[iPhoneme]; - if (bleftUnitName == bleftName) { - // Found a diphone -- add it to candidates - // DiphoneUnit diphoneUnit = new DiphoneUnit(leftNeighbour, unit); - // candidateUnitSet.add(diphoneUnit); - candidateUnitSet.add(leftNeighbour.index); - } - } - } - - // now create ArrayList of ViterbiCandidates from the candidateUnitSet, blacklisting along the way: - ArrayList candidates = new ArrayList(candidateUnitSet.size()); - for (int leftIndex : candidateUnitSet.toArray()) { - DiphoneUnit diphoneUnit = new DiphoneUnit(unitReader.units[leftIndex], unitReader.units[leftIndex + 1]); - ViterbiCandidate candidate = new ViterbiCandidate(diphoneTarget, diphoneUnit, targetCostFunction); - // Blacklisting: - if (blacklist.equals("")) { // no blacklist - candidates.add(candidate); - } else { // maybe exclude candidate - unitBasename = getFilename(diphoneUnit); - if (!blacklist.contains(unitBasename)) { - candidates.add(candidate); - } - } - } - - logger.debug("Preselected " + candidateUnitSet.size() + " diphone candidates for target " + target); - return candidates; - } - -} diff --git a/marytts-runtime/src/main/java/marytts/unitselection/data/FeatureFileReader.java b/marytts-runtime/src/main/java/marytts/unitselection/data/FeatureFileReader.java deleted file mode 100644 index 9ce3e6e136..0000000000 --- a/marytts-runtime/src/main/java/marytts/unitselection/data/FeatureFileReader.java +++ /dev/null @@ -1,218 +0,0 @@ -/** - * Copyright 2006 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.unitselection.data; - -import java.io.BufferedInputStream; -import java.io.DataInputStream; -import java.io.FileInputStream; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.channels.FileChannel; - -import marytts.exceptions.MaryConfigurationException; -import marytts.features.FeatureDefinition; -import marytts.features.FeatureVector; -import marytts.util.data.MaryHeader; - -public class FeatureFileReader { - protected MaryHeader hdr; - protected FeatureDefinition featureDefinition; - protected FeatureVector[] featureVectors; - - /** - * Get a feature file reader representing the given feature file. - * - * @param fileName - * the filename of a valid feature file. - * @return a feature file object representing the given file. - * @throws IOException - * if there was a problem reading the file - * @throws MaryConfigurationException - * if the file is not a valid feature file. - */ - public static FeatureFileReader getFeatureFileReader(String fileName) throws IOException, MaryConfigurationException { - int fileType = MaryHeader.peekFileType(fileName); - if (fileType == MaryHeader.UNITFEATS) - return new FeatureFileReader(fileName); - else if (fileType == MaryHeader.HALFPHONE_UNITFEATS) - return new HalfPhoneFeatureFileReader(fileName); - throw new MaryConfigurationException("File " + fileName + ": Type " + fileType + " is not a known unit feature file type"); - } - - /** - * Empty constructor; need to call load() separately when using this. - * - * @see load(String) - */ - public FeatureFileReader() { - } - - public FeatureFileReader(String fileName) throws IOException, MaryConfigurationException { - load(fileName); - } - - public void load(String fileName) throws IOException, MaryConfigurationException { - loadFromByteBuffer(fileName); - } - - protected void loadFromStream(String fileName) throws IOException, MaryConfigurationException { - /* Open the file */ - DataInputStream dis = null; - dis = new DataInputStream(new BufferedInputStream(new FileInputStream(fileName))); - - /* Load the Mary header */ - hdr = new MaryHeader(dis); - if (hdr.getType() != MaryHeader.UNITFEATS && hdr.getType() != MaryHeader.HALFPHONE_UNITFEATS) { - throw new IOException("File [" + fileName + "] is not a valid Mary feature file."); - } - featureDefinition = new FeatureDefinition(dis); - int numberOfUnits = dis.readInt(); - featureVectors = new FeatureVector[numberOfUnits]; - for (int i = 0; i < numberOfUnits; i++) { - featureVectors[i] = featureDefinition.readFeatureVector(i, dis); - } - - } - - protected void loadFromByteBuffer(String fileName) throws IOException, MaryConfigurationException { - /* Open the file */ - FileInputStream fis = new FileInputStream(fileName); - FileChannel fc = fis.getChannel(); - ByteBuffer bb = fc.map(FileChannel.MapMode.READ_ONLY, 0, fc.size()); - fis.close(); - - /* Load the Mary header */ - hdr = new MaryHeader(bb); - if (hdr.getType() != MaryHeader.UNITFEATS && hdr.getType() != MaryHeader.HALFPHONE_UNITFEATS) { - throw new MaryConfigurationException("File [" + fileName + "] is not a valid Mary feature file."); - } - featureDefinition = new FeatureDefinition(bb); - int numberOfUnits = bb.getInt(); - featureVectors = new FeatureVector[numberOfUnits]; - for (int i = 0; i < numberOfUnits; i++) { - featureVectors[i] = featureDefinition.readFeatureVector(i, bb); - } - - } - - /** - * Get the unit feature vector for the given unit index number. - * - * @param unitIndex - * the absolute index number of a unit in the database - * @return the corresponding feature vector - */ - public FeatureVector getFeatureVector(int unitIndex) { - return featureVectors[unitIndex]; - } - - /** - * Return a shallow copy of the array of feature vectors. - * - * @return a new array containing the internal feature vectors - */ - public FeatureVector[] getCopyOfFeatureVectors() { - return (FeatureVector[]) featureVectors.clone(); - } - - /** - * Return the internal array of feature vectors. - * - * @return the internal array of feature vectors. - */ - public FeatureVector[] getFeatureVectors() { - return featureVectors; - } - - /** - * feature vector mapping according to new feature definition Note: The new feature definition should be a subset of original - * feature definition - * - * @param newFeatureDefinition - * @return - */ - public FeatureVector[] featureVectorMapping(FeatureDefinition newFeatureDefinition) { - - if (!this.featureDefinition.contains(newFeatureDefinition)) { - throw new RuntimeException("the new feature definition is not a subset of original feature definition"); - } - - int numberOfFeatures = newFeatureDefinition.getNumberOfFeatures(); - int noByteFeatures = newFeatureDefinition.getNumberOfByteFeatures(); - int noShortFeatures = newFeatureDefinition.getNumberOfShortFeatures(); - int noContiniousFeatures = newFeatureDefinition.getNumberOfContinuousFeatures(); - - if (numberOfFeatures != (noByteFeatures + noShortFeatures + noContiniousFeatures)) { - throw new RuntimeException("The sum of byte, short and continious features are not equal to number of features"); - } - - String[] featureNames = new String[numberOfFeatures]; - for (int j = 0; j < numberOfFeatures; j++) { - featureNames[j] = newFeatureDefinition.getFeatureName(j); - } - int[] featureIndexes = featureDefinition.getFeatureIndexArray(featureNames); - FeatureVector[] newFV = new FeatureVector[this.getNumberOfUnits()]; - - for (int i = 0; i < this.getNumberOfUnits(); i++) { - - // create features array - byte[] byteFeatures = new byte[noByteFeatures]; - short[] shortFeatures = new short[noShortFeatures]; - float[] continiousFeatures = new float[noContiniousFeatures]; - - int countByteFeatures = 0; - int countShortFeatures = 0; - int countFloatFeatures = 0; - - for (int j = 0; j < featureIndexes.length; j++) { - if (newFeatureDefinition.isByteFeature(j)) { - byteFeatures[countByteFeatures++] = featureVectors[i].getByteFeature(featureIndexes[j]); - } else if (newFeatureDefinition.isShortFeature(j)) { - shortFeatures[countShortFeatures++] = featureVectors[i].getShortFeature(featureIndexes[j]); - } else if (newFeatureDefinition.isContinuousFeature(j)) { - continiousFeatures[countFloatFeatures++] = featureVectors[i].getContinuousFeature(featureIndexes[j]); - } - } - - newFV[i] = newFeatureDefinition.toFeatureVector(i, byteFeatures, shortFeatures, continiousFeatures); - } - - return newFV; - } - - /** - * Get the unit feature vector for the given unit. - * - * @param unit - * a unit in the database - * @return the corresponding feature vector - */ - public FeatureVector getFeatureVector(Unit unit) { - return featureVectors[unit.index]; - } - - public FeatureDefinition getFeatureDefinition() { - return featureDefinition; - } - - public int getNumberOfUnits() { - return (featureVectors.length); - } -} diff --git a/marytts-runtime/src/main/java/marytts/unitselection/data/FloatArrayDatagram.java b/marytts-runtime/src/main/java/marytts/unitselection/data/FloatArrayDatagram.java deleted file mode 100644 index 1036e6a049..0000000000 --- a/marytts-runtime/src/main/java/marytts/unitselection/data/FloatArrayDatagram.java +++ /dev/null @@ -1,67 +0,0 @@ -/** - * Copyright 2010 DFKI GmbH. All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as - * published by the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License along with this program. If not, see - * . - * - */ - -package marytts.unitselection.data; - -import java.io.BufferedOutputStream; -import java.io.ByteArrayOutputStream; -import java.io.DataOutput; -import java.io.DataOutputStream; -import java.io.IOException; - -import marytts.util.data.Datagram; - -/** - * Extension of Datagram to provide a float array instead of (actually alongside) a byte array - * - * @author steiner - * - */ -public class FloatArrayDatagram extends Datagram { - - private float[] floatData; - - public FloatArrayDatagram(long duration, float[] data) { - super(duration); - this.floatData = data; - } - - public float[] getFloatData() { - return floatData; - } - - /** - * Write this datagram to a random access file or data output stream. - * - * @throws IOException - */ - @Override - public void write(DataOutput raf) throws IOException { - - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - DataOutputStream dos = new DataOutputStream(baos); - BufferedOutputStream bos = new BufferedOutputStream(dos); - - dos.writeLong(duration); - dos.writeInt(floatData.length); - for (float fl : floatData) { - dos.writeFloat(fl); - } - - raf.write(baos.toByteArray()); - } - -} diff --git a/marytts-runtime/src/main/java/marytts/unitselection/data/HalfPhoneFeatureFileReader.java b/marytts-runtime/src/main/java/marytts/unitselection/data/HalfPhoneFeatureFileReader.java deleted file mode 100644 index fd3775e6f6..0000000000 --- a/marytts-runtime/src/main/java/marytts/unitselection/data/HalfPhoneFeatureFileReader.java +++ /dev/null @@ -1,99 +0,0 @@ -/** - * Copyright 2006 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.unitselection.data; - -import java.io.BufferedInputStream; -import java.io.DataInputStream; -import java.io.FileInputStream; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.channels.FileChannel; - -import marytts.exceptions.MaryConfigurationException; -import marytts.features.FeatureDefinition; -import marytts.features.FeatureVector; -import marytts.util.data.MaryHeader; - -public class HalfPhoneFeatureFileReader extends FeatureFileReader { - protected FeatureDefinition leftWeights; - protected FeatureDefinition rightWeights; - - public HalfPhoneFeatureFileReader() { - super(); - } - - public HalfPhoneFeatureFileReader(String fileName) throws IOException, MaryConfigurationException { - super(fileName); - } - - @Override - protected void loadFromStream(String fileName) throws IOException, MaryConfigurationException { - /* Open the file */ - DataInputStream dis = null; - dis = new DataInputStream(new BufferedInputStream(new FileInputStream(fileName))); - /* Load the Mary header */ - hdr = new MaryHeader(dis); - if (hdr.getType() != MaryHeader.HALFPHONE_UNITFEATS) { - throw new IOException("File [" + fileName + "] is not a valid Mary Halfphone Features file."); - } - leftWeights = new FeatureDefinition(dis); - rightWeights = new FeatureDefinition(dis); - assert leftWeights.featureEquals(rightWeights) : "Halfphone unit feature file contains incompatible feature definitions for left and right units -- this should not happen!"; - featureDefinition = leftWeights; // one of them, for super class - int numberOfUnits = dis.readInt(); - featureVectors = new FeatureVector[numberOfUnits]; - for (int i = 0; i < numberOfUnits; i++) { - featureVectors[i] = featureDefinition.readFeatureVector(i, dis); - } - } - - @Override - protected void loadFromByteBuffer(String fileName) throws IOException, MaryConfigurationException { - /* Open the file */ - FileInputStream fis = new FileInputStream(fileName); - FileChannel fc = fis.getChannel(); - ByteBuffer bb = fc.map(FileChannel.MapMode.READ_ONLY, 0, fc.size()); - fis.close(); - - /* Load the Mary header */ - hdr = new MaryHeader(bb); - if (hdr.getType() != MaryHeader.HALFPHONE_UNITFEATS) { - throw new MaryConfigurationException("File [" + fileName + "] is not a valid Mary Halfphone Features file."); - } - leftWeights = new FeatureDefinition(bb); - rightWeights = new FeatureDefinition(bb); - assert leftWeights.featureEquals(rightWeights) : "Halfphone unit feature file contains incompatible feature definitions for left and right units -- this should not happen!"; - featureDefinition = leftWeights; // one of them, for super class - int numberOfUnits = bb.getInt(); - featureVectors = new FeatureVector[numberOfUnits]; - for (int i = 0; i < numberOfUnits; i++) { - featureVectors[i] = featureDefinition.readFeatureVector(i, bb); - } - } - - public FeatureDefinition getLeftWeights() { - return leftWeights; - } - - public FeatureDefinition getRightWeights() { - return rightWeights; - } - -} diff --git a/marytts-runtime/src/main/java/marytts/unitselection/data/HnmDatagram.java b/marytts-runtime/src/main/java/marytts/unitselection/data/HnmDatagram.java deleted file mode 100644 index 35ca5bd246..0000000000 --- a/marytts-runtime/src/main/java/marytts/unitselection/data/HnmDatagram.java +++ /dev/null @@ -1,149 +0,0 @@ -/** - * Copyright 2006 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.unitselection.data; - -import java.io.ByteArrayInputStream; -import java.io.DataInputStream; -import java.io.DataOutput; -import java.io.EOFException; -import java.io.IOException; -import java.io.RandomAccessFile; -import java.nio.ByteBuffer; - -import marytts.signalproc.sinusoidal.hntm.analysis.HntmSpeechFrame; -import marytts.util.data.Datagram; - -/** - * A datagram that encapsulates a harmonics plus noise modelled speech frame - * - * @author Oytun Türk - * - */ -public class HnmDatagram extends Datagram { - - public HntmSpeechFrame frame; // Hnm parameters for a speech frame - - /** - * Construct a HNM datagram. - * - * @param duration - * the duration, in samples, of the data represented by this datagram - * @param frame - * the parameters of HNM for a speech frame. - */ - public HnmDatagram(long setDuration, HntmSpeechFrame frame) { - super(setDuration); - - this.frame = new HntmSpeechFrame(frame); - } - - /** - * Constructor which pops a datagram from a random access file. - * - * @param raf - * the random access file to pop the datagram from. - * - * @throws IOException - * @throws EOFException - */ - public HnmDatagram(RandomAccessFile raf, int noiseModel) throws IOException, EOFException { - super(raf.readLong()); // duration - int len = raf.readInt(); - if (len < 0) { - throw new IOException("Can't create a datagram with a negative data size [" + len + "]."); - } - if (len < 4 * 3) { - throw new IOException("Hnm with waveform noise datagram too short (len=" + len - + "): cannot be shorter than the space needed for first three Hnm parameters (4*3)"); - } - - // For speed concerns, read into a byte[] first: - byte[] buf = new byte[len]; - raf.readFully(buf); - DataInputStream dis = new DataInputStream(new ByteArrayInputStream(buf)); - - frame = new HntmSpeechFrame(dis, noiseModel); - } - - /** - * Constructor which pops a datagram from a byte buffer. - * - * @param bb - * the byte buffer to pop the datagram from. - * - * @throws IOException - * @throws EOFException - */ - public HnmDatagram(ByteBuffer bb, int noiseModel) throws IOException, EOFException { - super(bb.getLong()); // duration - int len = bb.getInt(); - if (len < 0) { - throw new IOException("Can't create a datagram with a negative data size [" + len + "]."); - } - if (len < 4 * 3) { - throw new IOException("Hnm with waveform noise datagram too short (len=" + len - + "): cannot be shorter than the space needed for first three Hnm parameters (4*3)"); - } - - frame = new HntmSpeechFrame(bb, noiseModel); - } - - /** - * Get the length, in bytes, of the datagram's data field. - */ - public int getLength() { - return frame.getLength(); - } - - /** - * Get the sinusoidal speech frame - * - * @return frame - */ - public HntmSpeechFrame getFrame() { - return frame; - } - - /** - * Write this datagram to a random access file or data output stream. - */ - public void write(DataOutput out) throws IOException { - out.writeLong(duration); - out.writeInt(getLength()); - - frame.write(out); - } - - /** - * Tests if this datagram is equal to another datagram. - */ - public boolean equals(Datagram other) { - if (!(other instanceof HnmDatagram)) - return false; - HnmDatagram otherHnm = (HnmDatagram) other; - if (this.duration != otherHnm.duration) - return false; - if (!this.frame.equals(otherHnm.frame)) - return false; - - return true; - } - -} diff --git a/marytts-runtime/src/main/java/marytts/unitselection/data/HnmTimelineReader.java b/marytts-runtime/src/main/java/marytts/unitselection/data/HnmTimelineReader.java deleted file mode 100644 index b9c307b1be..0000000000 --- a/marytts-runtime/src/main/java/marytts/unitselection/data/HnmTimelineReader.java +++ /dev/null @@ -1,248 +0,0 @@ -/** - * Copyright 2006 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.unitselection.data; - -import java.io.ByteArrayInputStream; -import java.io.DataOutputStream; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.LinkedList; -import java.util.Properties; - -import javax.sound.sampled.UnsupportedAudioFileException; - -import marytts.exceptions.MaryConfigurationException; -import marytts.signalproc.adaptation.prosody.BasicProsodyModifierParams; -import marytts.signalproc.sinusoidal.hntm.analysis.FrameNoisePartWaveform; -import marytts.signalproc.sinusoidal.hntm.analysis.HntmAnalyzerParams; -import marytts.signalproc.sinusoidal.hntm.analysis.HntmSpeechFrame; -import marytts.signalproc.sinusoidal.hntm.analysis.HntmSpeechSignal; -import marytts.signalproc.sinusoidal.hntm.synthesis.HntmSynthesizedSignal; -import marytts.signalproc.sinusoidal.hntm.synthesis.HntmSynthesizer; -import marytts.signalproc.sinusoidal.hntm.synthesis.HntmSynthesizerParams; -import marytts.util.data.Datagram; -import marytts.util.io.FileUtils; -import marytts.util.math.ArrayUtils; -import marytts.util.math.MathUtils; - -/** - * A reader class for the harmonics plus noise timeline file. - * - * @author Oytun Türk - * - */ -public class HnmTimelineReader extends TimelineReader { - public HntmAnalyzerParams analysisParams; - - public HnmTimelineReader(String fileName) throws IOException, MaryConfigurationException { - super(); - load(fileName); - } - - protected void load(String fileName) throws IOException, MaryConfigurationException { - super.load(fileName); - // Now make sense of the processing header - Properties props = new Properties(); - ByteArrayInputStream bais = new ByteArrayInputStream(procHdr.getString().getBytes("latin1")); - props.load(bais); - ensurePresent(props, "hnm.noiseModel"); - - analysisParams = new HntmAnalyzerParams(); - - analysisParams.noiseModel = Integer.parseInt(props.getProperty("hnm.noiseModel")); - analysisParams.hnmPitchVoicingAnalyzerParams.numFilteringStages = Integer - .parseInt(props.getProperty("hnm.numFiltStages")); - analysisParams.hnmPitchVoicingAnalyzerParams.medianFilterLength = Integer - .parseInt(props.getProperty("hnm.medianFiltLen")); - analysisParams.hnmPitchVoicingAnalyzerParams.movingAverageFilterLength = Integer.parseInt(props - .getProperty("hnm.maFiltLen")); - analysisParams.hnmPitchVoicingAnalyzerParams.cumulativeAmpThreshold = Float.parseFloat(props.getProperty("hnm.cumAmpTh")); - analysisParams.hnmPitchVoicingAnalyzerParams.maximumAmpThresholdInDB = Float - .parseFloat(props.getProperty("hnm.maxAmpTh")); - analysisParams.hnmPitchVoicingAnalyzerParams.harmonicDeviationPercent = Float.parseFloat(props - .getProperty("hnm.harmDevPercent")); - analysisParams.hnmPitchVoicingAnalyzerParams.sharpPeakAmpDiffInDB = Float.parseFloat(props - .getProperty("hnm.sharpPeakAmpDiff")); - analysisParams.hnmPitchVoicingAnalyzerParams.minimumTotalHarmonics = Integer.parseInt(props - .getProperty("hnm.minHarmonics")); - analysisParams.hnmPitchVoicingAnalyzerParams.maximumTotalHarmonics = Integer.parseInt(props - .getProperty("hnm.maxHarmonics")); - analysisParams.hnmPitchVoicingAnalyzerParams.minimumVoicedFrequencyOfVoicing = Float.parseFloat(props - .getProperty("hnm.minVoicedFreq")); - analysisParams.hnmPitchVoicingAnalyzerParams.maximumVoicedFrequencyOfVoicing = Float.parseFloat(props - .getProperty("hnm.maxVoicedFreq")); - analysisParams.hnmPitchVoicingAnalyzerParams.maximumFrequencyOfVoicingFinalShift = Float.parseFloat(props - .getProperty("hnm.maxFreqVoicingFinalShift")); - analysisParams.hnmPitchVoicingAnalyzerParams.neighsPercent = Float.parseFloat(props.getProperty("hnm.neighsPercent")); - analysisParams.harmonicPartCepstrumOrder = Integer.parseInt(props.getProperty("hnm.harmCepsOrder")); - analysisParams.regularizedCepstrumWarpingMethod = Integer.parseInt(props.getProperty("hnm.regCepWarpMethod")); - analysisParams.regularizedCepstrumLambdaHarmonic = Float.parseFloat(props.getProperty("hnm.regCepsLambda")); - analysisParams.noisePartLpOrder = Integer.parseInt(props.getProperty("hnm.noiseLpOrder")); - analysisParams.preemphasisCoefNoise = Float.parseFloat(props.getProperty("hnm.preCoefNoise")); - analysisParams.hpfBeforeNoiseAnalysis = Boolean.parseBoolean(props.getProperty("hnm.hpfBeforeNoiseAnalysis")); - analysisParams.numPeriodsHarmonicsExtraction = Float.parseFloat(props.getProperty("hnm.harmNumPer")); - } - - private void ensurePresent(Properties props, String key) throws MaryConfigurationException { - if (!props.containsKey(key)) - throw new MaryConfigurationException("Processing header does not contain required field '" + key + "'"); - } - - /** - * {@inheritDoc} - */ - @Override - protected Datagram getNextDatagram(ByteBuffer bb) { - - Datagram d = null; - - /* If the end of the datagram zone is reached, refuse to read */ - if (bb.position() == bb.limit()) { - // throw new IndexOutOfBoundsException( "Time out of bounds: you are trying to read a datagram at" + - // " a time which is bigger than the total timeline duration." ); - return null; - } - /* Else, pop the datagram out of the file */ - try { - d = new HnmDatagram(bb, analysisParams.noiseModel); - } - /* Detect a possible EOF encounter */ - catch (IOException e) { - return null; - } - - return d; - } - - private void testSynthesizeFromDatagrams(LinkedList datagrams, int startIndex, int endIndex, - DataOutputStream output) throws IOException { - HntmSynthesizer s = new HntmSynthesizer(); - // TODO: These should come from timeline and user choices... - HntmAnalyzerParams hnmAnalysisParams = new HntmAnalyzerParams(); - HntmSynthesizerParams synthesisParams = new HntmSynthesizerParams(); - BasicProsodyModifierParams pmodParams = new BasicProsodyModifierParams(); - int samplingRateInHz = this.getSampleRate(); - - int totalFrm = 0; - int i; - float originalDurationInSeconds = 0.0f; - float deltaTimeInSeconds; - - for (i = startIndex; i <= endIndex; i++) { - HnmDatagram datagram; - try { - datagram = datagrams.get(i); - } catch (IndexOutOfBoundsException e) { - throw e; - } - if (datagram != null && datagram instanceof HnmDatagram) { - totalFrm++; - // deltaTimeInSeconds = SignalProcUtils.sample2time(((HnmDatagram)datagrams.get(i)).getDuration(), - // samplingRateInHz); - deltaTimeInSeconds = datagram.frame.deltaAnalysisTimeInSeconds; - originalDurationInSeconds += deltaTimeInSeconds; - } - } - - HntmSpeechSignal hnmSignal = new HntmSpeechSignal(totalFrm, samplingRateInHz, originalDurationInSeconds); - - int frameCount = 0; - float tAnalysisInSeconds = 0.0f; - for (i = startIndex; i <= endIndex; i++) { - HnmDatagram datagram; - try { - datagram = datagrams.get(i); - } catch (IndexOutOfBoundsException e) { - throw e; - } - if (datagram != null && datagram instanceof HnmDatagram) { - // tAnalysisInSeconds += SignalProcUtils.sample2time(((HnmDatagram)datagrams.get(i)).getDuration(), - // samplingRateInHz); - tAnalysisInSeconds += datagram.getFrame().deltaAnalysisTimeInSeconds; - - if (frameCount < totalFrm) { - hnmSignal.frames[frameCount] = new HntmSpeechFrame(datagram.getFrame()); - hnmSignal.frames[frameCount].tAnalysisInSeconds = tAnalysisInSeconds; - frameCount++; - } - } - } - - HntmSynthesizedSignal ss = null; - if (totalFrm > 0) { - ss = s.synthesize(hnmSignal, null, null, pmodParams, null, hnmAnalysisParams, synthesisParams); - FileUtils.writeBinaryFile(ArrayUtils.copyDouble2Short(ss.output), output); - if (ss.output != null) { - ss.output = MathUtils.multiply(ss.output, 1.0 / 32768.0); // why is this done here? - } - } - return; - } - - /** - * Dump audio from HNM timeline to a series big-endian raw audio files in chunks of Datagrams (clusterSize). Run this - * with - * - * 
-	 * -ea - Xmx2gb
-	 *
- * - * @param args - *
    - *
  1. path to timeline_hnm.mry file
    2. - *
  2. path to dump output files
    3. - *
- * @throws IOException - */ - public static void main(String[] args) throws UnsupportedAudioFileException, IOException, MaryConfigurationException { - HnmTimelineReader h = new HnmTimelineReader(args[0]); - - LinkedList datagrams = new LinkedList(); - int count = 0; - long startDatagramTime = 0; - int numDatagrams = (int) h.numDatagrams; - // long numDatagrams = 2000; - - Datagram[] rawDatagrams = h.getDatagrams(0l, numDatagrams, h.getSampleRate(), null); - for (int i = 0; i < rawDatagrams.length; i++) { - HnmDatagram d = (HnmDatagram) rawDatagrams[i]; - datagrams.add(d); - count++; - System.out.println("Datagram " + String.valueOf(count) + "Noise waveform size=" - + ((FrameNoisePartWaveform) (((HnmDatagram) d).frame.n)).waveform().length); - } - - int clusterSize = 1000; - int numClusters = (int) Math.floor((numDatagrams) / ((double) clusterSize) + 0.5); - int startIndex, endIndex; - for (int i = 0; i < numClusters; i++) { - DataOutputStream output = new DataOutputStream(new FileOutputStream( - new File(String.format("%s_%06d.bin", args[1], i)))); - startIndex = (int) (i * clusterSize); - endIndex = (int) Math.min((i + 1) * clusterSize - 1, numDatagrams - 1); - h.testSynthesizeFromDatagrams(datagrams, startIndex, endIndex, output); - System.out.println("Timeline cluster " + String.valueOf(i + 1) + " of " + String.valueOf(numClusters) - + " synthesized..."); - output.close(); - } - } -} diff --git a/marytts-runtime/src/main/java/marytts/unitselection/data/LPCDatagram.java b/marytts-runtime/src/main/java/marytts/unitselection/data/LPCDatagram.java deleted file mode 100644 index ed52c357b1..0000000000 --- a/marytts-runtime/src/main/java/marytts/unitselection/data/LPCDatagram.java +++ /dev/null @@ -1,229 +0,0 @@ -/** - * Copyright 2006 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.unitselection.data; - -import java.io.ByteArrayInputStream; -import java.io.DataInputStream; -import java.io.DataOutput; -import java.io.EOFException; -import java.io.IOException; -import java.io.RandomAccessFile; -import java.nio.ByteBuffer; - -import marytts.util.data.Datagram; -import marytts.util.io.General; - -public class LPCDatagram extends Datagram { - protected short[] quantizedCoeffs; - protected byte[] quantizedResidual; - - /** - * Construct an LPC datagram from quantized data. - * - * @param duration - * the duration, in samples, of the data represented by this datagram - * @param quantizedCoeffs - * the quantized LPC coefficients - * @param quantizedResidual - * the quantized residual - */ - public LPCDatagram(long setDuration, short[] quantizedCoeffs, byte[] quantizedResidual) { - super(setDuration); - this.quantizedCoeffs = quantizedCoeffs; - this.quantizedResidual = quantizedResidual; - } - - /** - * Construct an LPC datagram from unquantized data. - * - * @param duration - * the duration, in samples, of the data represented by this datagram - * @param coeffs - * the (unquantized) LPC coefficients - * @param residual - * the (unquantized) residual - */ - public LPCDatagram(long setDuration, float[] coeffs, short[] residual, float lpcMin, float lpcRange) { - super(setDuration); - this.quantizedCoeffs = General.quantize(coeffs, lpcMin, lpcRange); - this.quantizedResidual = General.shortToUlaw(residual); - } - - /** - * Constructor which pops a datagram from a random access file. - * - * @param raf - * the random access file to pop the datagram from. - * - * @throws IOException - * @throws EOFException - */ - public LPCDatagram(RandomAccessFile raf, int lpcOrder) throws IOException, EOFException { - super(raf.readLong()); // duration - int len = raf.readInt(); - if (len < 0) { - throw new IOException("Can't create a datagram with a negative data size [" + len + "]."); - } - if (len < 2 * lpcOrder) { - throw new IOException("LPC datagram too short (len=" + len - + "): cannot be shorter than the space needed for lpc coefficients (2*" + lpcOrder + ")"); - } - // For speed concerns, read into a byte[] first: - byte[] buf = new byte[len]; - raf.readFully(buf); - DataInputStream dis = new DataInputStream(new ByteArrayInputStream(buf)); - - int residualLength = len - 2 * lpcOrder; - quantizedCoeffs = new short[lpcOrder]; - quantizedResidual = new byte[residualLength]; - for (int i = 0; i < lpcOrder; i++) { - quantizedCoeffs[i] = dis.readShort(); - } - System.arraycopy(buf, 2 * lpcOrder, quantizedResidual, 0, residualLength); - } - - /** - * Constructor which pops a datagram from a byte buffer. - * - * @param bb - * the byte buffer to pop the datagram from. - * - * @throws IOException - * @throws EOFException - */ - public LPCDatagram(ByteBuffer bb, int lpcOrder) throws IOException, EOFException { - super(bb.getLong()); // duration - int len = bb.getInt(); - if (len < 0) { - throw new IOException("Can't create a datagram with a negative data size [" + len + "]."); - } - if (len < 2 * lpcOrder) { - throw new IOException("LPC datagram too short (len=" + len - + "): cannot be shorter than the space needed for lpc coefficients (2*" + lpcOrder + ")"); - } - - int residualLength = len - 2 * lpcOrder; - quantizedCoeffs = new short[lpcOrder]; - quantizedResidual = new byte[residualLength]; - for (int i = 0; i < lpcOrder; i++) { - quantizedCoeffs[i] = bb.getShort(); - } - bb.get(quantizedResidual); - } - - /** - * Get the length, in bytes, of the datagram's data field. - */ - public int getLength() { - return 2 * quantizedCoeffs.length + quantizedResidual.length; - } - - /** - * Get the LPC order, i.e. the number of LPC coefficients. - * - * @return the lpc order - * @see #getQuantizedCoeffs() - * @see #getCoeffs() - */ - public int lpcOrder() { - return quantizedCoeffs.length; - } - - /** - * Get the quantized lpc coefficients - * - * @return an array of shorts, length lpcOrder() - * @see #lpcOrder() - * @see #getCoeffs() - */ - public short[] getQuantizedCoeffs() { - return quantizedCoeffs; - } - - /** - * Get the quantized residual. - * - * @return an array of bytes - */ - public byte[] getQuantizedResidual() { - return quantizedResidual; - } - - /** - * Get the LPC coefficients, unquantized using the given lpc min and range values. - * - * @param lpcMin - * the lpc minimum - * @param lpcRange - * the lpc range - * @return an array of floats, length lpcOrder() - * @see #lpcOrder() - * @see #getQuantizedCoeffs() - */ - public float[] getCoeffs(float lpcMin, float lpcRange) { - return General.unQuantize(quantizedCoeffs, lpcMin, lpcRange); - } - - /** - * Get the unquantized residual - * - * @return an array of shorts - */ - public short[] getResidual() { - return General.ulawToShort(quantizedResidual); - } - - /** - * Write this datagram to a random access file or data output stream. - */ - public void write(DataOutput out) throws IOException { - out.writeLong(duration); - out.writeInt(getLength()); - for (int i = 0; i < quantizedCoeffs.length; i++) { - out.writeShort(quantizedCoeffs[i]); - } - out.write(quantizedResidual); - } - - /** - * Tests if this datagram is equal to another datagram. - */ - public boolean equals(Datagram other) { - if (!(other instanceof LPCDatagram)) - return false; - LPCDatagram otherLPC = (LPCDatagram) other; - if (this.duration != otherLPC.duration) - return false; - if (this.quantizedCoeffs.length != otherLPC.quantizedCoeffs.length) - return false; - if (this.quantizedResidual.length != otherLPC.quantizedResidual.length) - return false; - for (int i = 0; i < this.quantizedCoeffs.length; i++) { - if (this.quantizedCoeffs[i] != otherLPC.quantizedCoeffs[i]) - return false; - } - for (int i = 0; i < this.quantizedResidual.length; i++) { - if (this.quantizedResidual[i] != otherLPC.quantizedResidual[i]) - return false; - } - return true; - } - -} diff --git a/marytts-runtime/src/main/java/marytts/unitselection/data/LPCTimelineReader.java b/marytts-runtime/src/main/java/marytts/unitselection/data/LPCTimelineReader.java deleted file mode 100644 index 3fcb84e961..0000000000 --- a/marytts-runtime/src/main/java/marytts/unitselection/data/LPCTimelineReader.java +++ /dev/null @@ -1,100 +0,0 @@ -/** - * Copyright 2006 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.unitselection.data; - -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.Properties; - -import marytts.exceptions.MaryConfigurationException; -import marytts.util.data.Datagram; - -public class LPCTimelineReader extends TimelineReader { - protected int lpcOrder; - protected float lpcMin; - protected float lpcRange; - - public LPCTimelineReader(String fileName) throws IOException, MaryConfigurationException { - super(); - load(fileName); - } - - @Override - protected void load(String fileName) throws IOException, MaryConfigurationException { - super.load(fileName); - // Now make sense of the processing header - Properties props = new Properties(); - ByteArrayInputStream bais = new ByteArrayInputStream(procHdr.getString().getBytes("latin1")); - props.load(bais); - ensurePresent(props, "lpc.order"); - lpcOrder = Integer.parseInt(props.getProperty("lpc.order")); - ensurePresent(props, "lpc.min"); - lpcMin = Float.parseFloat(props.getProperty("lpc.min")); - ensurePresent(props, "lpc.range"); - lpcRange = Float.parseFloat(props.getProperty("lpc.range")); - } - - private void ensurePresent(Properties props, String key) throws IOException { - if (!props.containsKey(key)) - throw new IOException("Processing header does not contain required field '" + key + "'"); - - } - - public int getLPCOrder() { - return lpcOrder; - } - - public float getLPCMin() { - return lpcMin; - } - - public float getLPCRange() { - return lpcRange; - } - - /** - * Read and return the upcoming datagram. - * - * @return the current datagram, or null if EOF was encountered; internally updates the time pointer. - * - * @throws IOException - */ - @Override - protected Datagram getNextDatagram(ByteBuffer bb) { - - Datagram d = null; - - /* If the end of the datagram zone is reached, gracefully refuse to read */ - if (bb.position() == timeIdxBytePos) - return (null); - /* Else, pop the datagram out of the file */ - try { - d = new LPCDatagram(bb, lpcOrder); - } - /* Detect a possible EOF encounter */ - catch (IOException e) { - return null; - } - - return (d); - } - -} diff --git a/marytts-runtime/src/main/java/marytts/unitselection/data/MCepDatagram.java b/marytts-runtime/src/main/java/marytts/unitselection/data/MCepDatagram.java deleted file mode 100644 index 167bc917ca..0000000000 --- a/marytts-runtime/src/main/java/marytts/unitselection/data/MCepDatagram.java +++ /dev/null @@ -1,175 +0,0 @@ -/** - * Copyright 2006 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.unitselection.data; - -import java.io.ByteArrayInputStream; -import java.io.DataInputStream; -import java.io.DataOutput; -import java.io.EOFException; -import java.io.IOException; -import java.io.RandomAccessFile; -import java.nio.ByteBuffer; - -import marytts.util.data.Datagram; - -public class MCepDatagram extends Datagram { - - protected float[] coeffs; - - /** - * Construct a MCep datagram from a float vector. - * - * @param duration - * the duration, in samples, of the data represented by this datagram - * @param coeffs - * the array of Mel-Cepstrum coefficients. - */ - public MCepDatagram(long setDuration, float[] coeffs) { - super(setDuration); - this.coeffs = coeffs; - } - - /** - * Constructor which pops a datagram from a random access file. - * - * @param raf - * the random access file to pop the datagram from. - * - * @throws IOException - * @throws EOFException - */ - public MCepDatagram(RandomAccessFile raf, int order) throws IOException, EOFException { - super(raf.readLong()); // duration - int len = raf.readInt(); - if (len < 0) { - throw new IOException("Can't create a datagram with a negative data size [" + len + "]."); - } - if (len < 4 * order) { - throw new IOException("Mel-Cepstrum datagram too short (len=" + len - + "): cannot be shorter than the space needed for Mel-Cepstrum coefficients (4*" + order + ")"); - } - // For speed concerns, read into a byte[] first: - byte[] buf = new byte[len]; - raf.readFully(buf); - DataInputStream dis = new DataInputStream(new ByteArrayInputStream(buf)); - - coeffs = new float[order]; - for (int i = 0; i < order; i++) { - coeffs[i] = dis.readFloat(); - } - } - - /** - * Constructor which pops a datagram from a byte buffer. - * - * @param raf - * the byte buffer to pop the datagram from. - * - * @throws IOException - * @throws EOFException - */ - public MCepDatagram(ByteBuffer bb, int order) throws IOException, EOFException { - super(bb.getLong()); // duration - int len = bb.getInt(); - if (len < 0) { - throw new IOException("Can't create a datagram with a negative data size [" + len + "]."); - } - if (len < 4 * order) { - throw new IOException("Mel-Cepstrum datagram too short (len=" + len - + "): cannot be shorter than the space needed for Mel-Cepstrum coefficients (4*" + order + ")"); - } - coeffs = new float[order]; - for (int i = 0; i < order; i++) { - coeffs[i] = bb.getFloat(); - } - } - - /** - * Get the length, in bytes, of the datagram's data field. - */ - public int getLength() { - return 4 * coeffs.length; - } - - /** - * Get the order, i.e. the number of MEl-Cepstrum coefficients. - * - * @return the order - * @see #getCoeffs() - */ - public int order() { - return coeffs.length; - } - - /** - * Get the array of Mel-Cepstrum coefficients. - */ - public float[] getCoeffs() { - return coeffs; - } - - /** - * Get the array of Mel-Cepstrum coefficients. - */ - public double[] getCoeffsAsDouble() { - double[] ret = new double[coeffs.length]; - for (int i = 0; i < coeffs.length; i++) { - ret[i] = (double) (coeffs[i]); - } - return (ret); - } - - /** - * Get a particular Mel-Cepstrum coefficient. - */ - public float getCoeff(int i) { - return coeffs[i]; - } - - /** - * Write this datagram to a random access file or data output stream. - */ - public void write(DataOutput out) throws IOException { - out.writeLong(duration); - out.writeInt(getLength()); - for (int i = 0; i < coeffs.length; i++) { - out.writeFloat(coeffs[i]); - } - } - - /** - * Tests if this datagram is equal to another datagram. - */ - public boolean equals(Datagram other) { - if (!(other instanceof MCepDatagram)) - return false; - MCepDatagram otherMCep = (MCepDatagram) other; - if (this.duration != otherMCep.duration) - return false; - if (this.coeffs.length != otherMCep.coeffs.length) - return false; - for (int i = 0; i < this.coeffs.length; i++) { - if (this.coeffs[i] != otherMCep.coeffs[i]) - return false; - } - return true; - } - -} diff --git a/marytts-runtime/src/main/java/marytts/unitselection/data/MCepTimelineReader.java b/marytts-runtime/src/main/java/marytts/unitselection/data/MCepTimelineReader.java deleted file mode 100644 index cc9e9516c0..0000000000 --- a/marytts-runtime/src/main/java/marytts/unitselection/data/MCepTimelineReader.java +++ /dev/null @@ -1,85 +0,0 @@ -/** - * Copyright 2006 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.unitselection.data; - -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.Properties; - -import marytts.exceptions.MaryConfigurationException; -import marytts.util.data.Datagram; - -public class MCepTimelineReader extends TimelineReader { - protected int order; - - public MCepTimelineReader(String fileName) throws IOException, MaryConfigurationException { - super(); - load(fileName); - } - - @Override - protected void load(String fileName) throws IOException, MaryConfigurationException { - super.load(fileName); - // Now make sense of the processing header - Properties props = new Properties(); - ByteArrayInputStream bais = new ByteArrayInputStream(procHdr.getString().getBytes("latin1")); - props.load(bais); - ensurePresent(props, "mcep.order"); - order = Integer.parseInt(props.getProperty("mcep.order")); - } - - private void ensurePresent(Properties props, String key) throws IOException { - if (!props.containsKey(key)) - throw new IOException("Processing header does not contain required field '" + key + "'"); - - } - - public int getOrder() { - return order; - } - - /** - * Read and return the upcoming datagram. - * - * @return the current datagram, or null if EOF was encountered; internally updates the time pointer. - * - */ - @Override - protected Datagram getNextDatagram(ByteBuffer bb) { - - Datagram d = null; - - /* If the end of the datagram zone is reached, gracefully refuse to read */ - if (bb.position() == timeIdxBytePos) - return (null); - /* Else, pop the datagram out of the file */ - try { - d = new MCepDatagram(bb, order); - } - /* Detect a possible EOF encounter */ - catch (IOException e) { - return null; - } - - return (d); - } - -} diff --git a/marytts-runtime/src/main/java/marytts/unitselection/data/SCostFileReader.java b/marytts-runtime/src/main/java/marytts/unitselection/data/SCostFileReader.java deleted file mode 100644 index 5bd02e274b..0000000000 --- a/marytts-runtime/src/main/java/marytts/unitselection/data/SCostFileReader.java +++ /dev/null @@ -1,140 +0,0 @@ -/** - * Portions Copyright 2006 DFKI GmbH. - * Portions Copyright 2001 Sun Microsystems, Inc. - * Portions Copyright 1999-2001 Language Technologies Institute, - * Carnegie Mellon University. - * All Rights Reserved. Use is subject to license terms. - * - * Permission is hereby granted, free of charge, to use and distribute - * this software and its documentation without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of this work, and to - * permit persons to whom this work is furnished to do so, subject to - * the following conditions: - * - * 1. The code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * 2. Any modifications must be clearly marked as such. - * 3. Original authors' names are not deleted. - * 4. The authors' names are not used to endorse or promote products - * derived from this software without specific prior written - * permission. - * - * DFKI GMBH AND THE CONTRIBUTORS TO THIS WORK DISCLAIM ALL WARRANTIES WITH - * REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL DFKI GMBH NOR THE - * CONTRIBUTORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL - * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR - * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS - * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF - * THIS SOFTWARE. - */ -package marytts.unitselection.data; - -import java.io.BufferedInputStream; -import java.io.DataInputStream; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.IOException; - -import marytts.exceptions.MaryConfigurationException; -import marytts.util.data.MaryHeader; - -/** - * sCost file reader - * - * @author sathish pammi - * - */ -public class SCostFileReader { - - private MaryHeader hdr = null; - private int numberOfUnits = 0; - private double[] sCost; - - /****************/ - /* CONSTRUCTORS */ - /****************/ - - /** - * Empty constructor; need to call load() separately. - * - * @see #load(String) - */ - public SCostFileReader() { - } - - /** - * Create a unit file reader from the given unit file - * - * @param fileName - * the unit file to read - * @throws IOException - * if a problem occurs while reading - */ - public SCostFileReader(String fileName) throws IOException, MaryConfigurationException { - load(fileName); - } - - /** - * Load the given unit file - * - * @param fileName - * the unit file to read - * @throws IOException - * if a problem occurs while reading - */ - public void load(String fileName) throws IOException, MaryConfigurationException { - /* Open the file */ - DataInputStream dis = null; - try { - dis = new DataInputStream(new BufferedInputStream(new FileInputStream(fileName))); - } catch (FileNotFoundException e) { - throw new RuntimeException("File [" + fileName + "] was not found."); - } - try { - /* Load the Mary header */ - hdr = new MaryHeader(dis); - if (hdr.getType() != MaryHeader.SCOST) { - throw new RuntimeException("File [" + fileName + "] is not a valid Mary Units file."); - } - /* Read the number of units */ - numberOfUnits = dis.readInt(); - if (numberOfUnits < 0) { - throw new RuntimeException("File [" + fileName + "] has a negative number of units. Aborting."); - } - - sCost = new double[numberOfUnits]; - /* Read the start times and durations */ - for (int i = 0; i < numberOfUnits; i++) { - sCost[i] = dis.readFloat(); - } - } catch (IOException e) { - throw new RuntimeException("Reading the Mary header from file [" + fileName + "] failed.", e); - } - - } - - /*****************/ - /* OTHER METHODS */ - /*****************/ - - /** - * Get the number of units in the file. - * - * @return The number of units. - */ - public int getNumberOfUnits() { - return (numberOfUnits); - } - - /** - * Get sCost for a unit index - * - * @return sCost - */ - public double getSCost(int index) { - return (this.sCost[index]); - } - -} diff --git a/marytts-runtime/src/main/java/marytts/unitselection/data/Sentence.java b/marytts-runtime/src/main/java/marytts/unitselection/data/Sentence.java deleted file mode 100644 index f026ff0168..0000000000 --- a/marytts-runtime/src/main/java/marytts/unitselection/data/Sentence.java +++ /dev/null @@ -1,63 +0,0 @@ -/** - * Copyright 2010 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ - -package marytts.unitselection.data; - -import java.util.Iterator; - -/** - * This class represents the section of a feature file which constitutes a sentence. - * - * @author marc - * - */ -public class Sentence implements Iterable { - - private FeatureFileReader features; - private int firstUnitIndex; - private int lastUnitIndex; - - public Sentence(FeatureFileReader features, int firstUnitIndex, int lastUnitIndex) { - this.features = features; - this.firstUnitIndex = firstUnitIndex; - this.lastUnitIndex = lastUnitIndex; - } - - public int getFirstUnitIndex() { - return firstUnitIndex; - } - - public int getLastUnitIndex() { - return lastUnitIndex; - } - - public Iterator iterator() { - return new SyllableIterator(features, firstUnitIndex, lastUnitIndex); - } - - @Override - public boolean equals(Object o) { - if (!(o instanceof Sentence)) { - return false; - } - Sentence other = (Sentence) o; - return features.equals(other.features) && firstUnitIndex == other.firstUnitIndex && lastUnitIndex == other.lastUnitIndex; - } -} diff --git a/marytts-runtime/src/main/java/marytts/unitselection/data/SentenceIterator.java b/marytts-runtime/src/main/java/marytts/unitselection/data/SentenceIterator.java deleted file mode 100644 index ed7e60f403..0000000000 --- a/marytts-runtime/src/main/java/marytts/unitselection/data/SentenceIterator.java +++ /dev/null @@ -1,152 +0,0 @@ -/** - * Copyright 2010 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ - -package marytts.unitselection.data; - -import java.util.Iterator; -import java.util.NoSuchElementException; - -import marytts.features.FeatureDefinition; -import marytts.features.FeatureVector; - -/** - * Iterator to provide the sentences in a given feature file, in sequence. - * - * @author marc - */ -public class SentenceIterator implements Iterator { - - private final FeatureFileReader features; - private final int fiSentenceStart; - private final int fiSentenceEnd; - private final int fiWordStart; - private final int fiWordEnd; - private final boolean isHalfphone; - private final int fiLR; - private final int fvLR_L; - private final int fvLR_R; - - private int i; - private int len; - private Sentence nextSentence = null; - - public SentenceIterator(FeatureFileReader features) { - this.features = features; - FeatureDefinition featureDefinition = features.getFeatureDefinition(); - fiSentenceStart = featureDefinition.getFeatureIndex("words_from_sentence_start"); - fiSentenceEnd = featureDefinition.getFeatureIndex("words_from_sentence_end"); - fiWordStart = featureDefinition.getFeatureIndex("segs_from_word_start"); - fiWordEnd = featureDefinition.getFeatureIndex("segs_from_word_end"); - String halfphoneFeature = "halfphone_lr"; - if (featureDefinition.hasFeature(halfphoneFeature)) { - isHalfphone = true; - fiLR = featureDefinition.getFeatureIndex(halfphoneFeature); - fvLR_L = featureDefinition.getFeatureValueAsByte(fiLR, "L"); - fvLR_R = featureDefinition.getFeatureValueAsByte(fiLR, "R"); - } else { - isHalfphone = false; - fiLR = fvLR_L = fvLR_R = 0; - } - - i = 0; - len = features.getNumberOfUnits(); - } - - public synchronized boolean hasNext() { - if (nextSentence == null) { - prepareNextSentence(); - } - return nextSentence != null; - } - - public synchronized Sentence next() { - if (nextSentence == null) { - prepareNextSentence(); - } - if (nextSentence == null) { - // no more sentences - throw new NoSuchElementException("no more sentences!"); - } - Sentence retval = nextSentence; - nextSentence = null; - return retval; - } - - public void remove() { - throw new UnsupportedOperationException("This iterator cannot remove sentences"); - } - - /** - * Find the next sentence in the feature file, if possible. - */ - private void prepareNextSentence() { - if (nextSentence != null) { - return; - } - if (i >= len) { - return; - } - // if we get here, then i is the index of a unit before or at a sentence start - while (i < len && !isSentenceStart(i)) { - i++; - } - if (i >= len) { - return; - } - int iSentenceStart = i; - while (i < len && !isSentenceEnd(i)) { - i++; - } - if (i >= len) { - return; - } - int iSentenceEnd = i; - nextSentence = new Sentence(features, iSentenceStart, iSentenceEnd); - } - - /** - * Check if the given unit index is a sentence start - * - * @param index - * the unit index - */ - private boolean isSentenceStart(int index) { - FeatureVector fv = features.getFeatureVector(index); - - return fv.getByteFeature(fiSentenceStart) == 0 // first word in sentence - && fv.getByteFeature(fiWordStart) == 0 // first segment in word - && (!isHalfphone || fv.getByteFeature(fiLR) == fvLR_L); // for halfphones, it's the left half - } - - /** - * Check if the given unit index is a sentence end - * - * @param index - * the unit index - */ - private boolean isSentenceEnd(int index) { - FeatureVector fv = features.getFeatureVector(index); - - return fv.getByteFeature(fiSentenceEnd) == 0 // last word in sentence - && fv.getByteFeature(fiWordEnd) == 0 // last segment in word - && (!isHalfphone || fv.getByteFeature(fiLR) == fvLR_R); // for halfphones, it's the right half - } - -} diff --git a/marytts-runtime/src/main/java/marytts/unitselection/data/Syllable.java b/marytts-runtime/src/main/java/marytts/unitselection/data/Syllable.java deleted file mode 100644 index afe14b211e..0000000000 --- a/marytts-runtime/src/main/java/marytts/unitselection/data/Syllable.java +++ /dev/null @@ -1,77 +0,0 @@ -/** - * Copyright 2010 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ - -package marytts.unitselection.data; - -import marytts.features.FeatureVector; - -/** - * This class represents the section of a feature file which constitutes a sentence. - * - * @author marc - * - */ -public class Syllable { - - private FeatureFileReader features; - private int firstUnitIndex; - private int lastUnitIndex; - - public Syllable(FeatureFileReader features, int firstUnitIndex, int lastUnitIndex) { - this.features = features; - this.firstUnitIndex = firstUnitIndex; - this.lastUnitIndex = lastUnitIndex; - } - - public int getFirstUnitIndex() { - return firstUnitIndex; - } - - public int getLastUnitIndex() { - return lastUnitIndex; - } - - /** - * Seek for the syllable nucleus (with feature "ph_vc" == "+") from first to last unit; if none is found, return the last unit - * in the syllable - * - * @return - */ - public int getSyllableNucleusIndex() { - int fiVowel = features.getFeatureDefinition().getFeatureIndex("ph_vc"); - byte fvVowel_Plus = features.getFeatureDefinition().getFeatureValueAsByte(fiVowel, "+"); - for (int i = firstUnitIndex; i <= lastUnitIndex; i++) { - FeatureVector fv = features.getFeatureVector(i); - if (fv.getByteFeature(fiVowel) == fvVowel_Plus) { - return i; - } - } - return lastUnitIndex; - } - - @Override - public boolean equals(Object o) { - if (!(o instanceof Syllable)) { - return false; - } - Syllable other = (Syllable) o; - return features.equals(other.features) && firstUnitIndex == other.firstUnitIndex && lastUnitIndex == other.lastUnitIndex; - } -} diff --git a/marytts-runtime/src/main/java/marytts/unitselection/data/SyllableIterator.java b/marytts-runtime/src/main/java/marytts/unitselection/data/SyllableIterator.java deleted file mode 100644 index 1532972449..0000000000 --- a/marytts-runtime/src/main/java/marytts/unitselection/data/SyllableIterator.java +++ /dev/null @@ -1,151 +0,0 @@ -/** - * Copyright 2009 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ - -package marytts.unitselection.data; - -import java.util.Iterator; -import java.util.NoSuchElementException; - -import marytts.features.FeatureDefinition; -import marytts.features.FeatureVector; - -/** - * @author marc - * - */ -public class SyllableIterator implements Iterator { - - private FeatureFileReader features; - private int fromUnitIndex; - private int toUnitIndex; - - private final int fiPhone; - private final byte fvPhone_0; - private final byte fvPhone_Silence; - private final int fiSylStart; - private final int fiSylEnd; - private final boolean isHalfphone; - private final int fiLR; - private final int fvLR_L; - private final int fvLR_R; - - private int i; - private Syllable nextSyllable = null; - - /** - * Create a syllable iterator over the given feature file, starting from the given fromUnitIndex and reaching up to (and - * including) the given toUnitIndex - * - * @param features - * @param fromUnitIndex - * @param toUnitIndex - */ - public SyllableIterator(FeatureFileReader features, int fromUnitIndex, int toUnitIndex) { - this.features = features; - this.fromUnitIndex = fromUnitIndex; - this.toUnitIndex = toUnitIndex; - - FeatureDefinition featureDefinition = features.getFeatureDefinition(); - fiPhone = featureDefinition.getFeatureIndex("phone"); - fvPhone_0 = featureDefinition.getFeatureValueAsByte(fiPhone, "0"); - fvPhone_Silence = featureDefinition.getFeatureValueAsByte(fiPhone, "_"); - fiSylStart = featureDefinition.getFeatureIndex("segs_from_syl_start"); - fiSylEnd = featureDefinition.getFeatureIndex("segs_from_syl_end"); - String halfphoneFeature = "halfphone_lr"; - if (featureDefinition.hasFeature(halfphoneFeature)) { - isHalfphone = true; - fiLR = featureDefinition.getFeatureIndex(halfphoneFeature); - fvLR_L = featureDefinition.getFeatureValueAsByte(fiLR, "L"); - fvLR_R = featureDefinition.getFeatureValueAsByte(fiLR, "R"); - } else { - isHalfphone = false; - fiLR = fvLR_L = fvLR_R = 0; - } - - i = fromUnitIndex; - } - - public synchronized boolean hasNext() { - if (nextSyllable == null) { - prepareNextSyllable(); - } - return nextSyllable != null; - } - - public synchronized Syllable next() { - if (nextSyllable == null) { - prepareNextSyllable(); - } - if (nextSyllable == null) { - // no more syllables - throw new NoSuchElementException("no more syllables!"); - } - Syllable retval = nextSyllable; - nextSyllable = null; - return retval; - } - - public void remove() { - throw new UnsupportedOperationException("This iterator cannot remove syllables"); - } - - private void prepareNextSyllable() { - if (nextSyllable != null) { - return; - } - if (i > toUnitIndex) { - return; - } - // if we get here, then i is the index of a unit before or at a syllable start - while (i <= toUnitIndex && !isSyllableStart(i)) { - i++; - } - if (i > toUnitIndex) { - return; - } - int iSyllableStart = i; - while (i <= toUnitIndex && !isSyllableEnd(i)) { - i++; - } - if (i > toUnitIndex) { - return; - } - int iSyllableEnd = i; - nextSyllable = new Syllable(features, iSyllableStart, iSyllableEnd); - } - - private boolean isSyllableStart(int index) { - FeatureVector fv = features.getFeatureVector(index); - - return fv.getByteFeature(fiPhone) != fvPhone_0 // not an edge unit - && fv.getByteFeature(fiPhone) != fvPhone_Silence // not silence - && fv.getByteFeature(fiSylStart) == 0 // first segment in syllable - && (!isHalfphone || fv.getByteFeature(fiLR) == fvLR_L); // if halfphone, it's the left half - } - - private boolean isSyllableEnd(int index) { - FeatureVector fv = features.getFeatureVector(index); - - return fv.getByteFeature(fiPhone) != fvPhone_0 // not an edge unit - && fv.getByteFeature(fiPhone) != fvPhone_Silence // not silence - && fv.getByteFeature(fiSylEnd) == 0 // last segment in syllable - && (!isHalfphone || fv.getByteFeature(fiLR) == fvLR_R); // if halfphone, it's the right half - } -} diff --git a/marytts-runtime/src/main/java/marytts/unitselection/data/TimelineReader.java b/marytts-runtime/src/main/java/marytts/unitselection/data/TimelineReader.java deleted file mode 100644 index 353c93b3dd..0000000000 --- a/marytts-runtime/src/main/java/marytts/unitselection/data/TimelineReader.java +++ /dev/null @@ -1,1304 +0,0 @@ -/** - * Portions Copyright 2006 DFKI GmbH. - * Portions Copyright 2001 Sun Microsystems, Inc. - * Portions Copyright 1999-2001 Language Technologies Institute, - * Carnegie Mellon University. - * All Rights Reserved. Use is subject to license terms. - * - * Permission is hereby granted, free of charge, to use and distribute - * this software and its documentation without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of this work, and to - * permit persons to whom this work is furnished to do so, subject to - * the following conditions: - * - * 1. The code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * 2. Any modifications must be clearly marked as such. - * 3. Original authors' names are not deleted. - * 4. The authors' names are not used to endorse or promote products - * derived from this software without specific prior written - * permission. - * - * DFKI GMBH AND THE CONTRIBUTORS TO THIS WORK DISCLAIM ALL WARRANTIES WITH - * REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL DFKI GMBH NOR THE - * CONTRIBUTORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL - * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR - * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS - * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF - * THIS SOFTWARE. - */ -package marytts.unitselection.data; - -import java.io.ByteArrayInputStream; -import java.io.DataInput; -import java.io.DataInputStream; -import java.io.IOException; -import java.io.RandomAccessFile; -import java.io.UTFDataFormatException; -import java.nio.BufferUnderflowException; -import java.nio.ByteBuffer; -import java.nio.MappedByteBuffer; -import java.nio.channels.FileChannel; -import java.util.ArrayList; -import java.util.Vector; - -import marytts.exceptions.MaryConfigurationException; -import marytts.util.MaryUtils; -import marytts.util.Pair; -import marytts.util.data.Datagram; -import marytts.util.data.MaryHeader; -import marytts.util.io.StreamUtils; - -/** - * The TimelineReader class provides an interface to read regularly or variably spaced datagrams from a Timeline data file in Mary - * format. - * - * @author sacha, marc - * - */ -public class TimelineReader { - protected MaryHeader maryHdr = null; // The standard Mary header - protected ProcHeader procHdr = null; // The processing info header - - protected Index idx = null; // A global time index for the variable-sized datagrams - - /* Some specific header fields: */ - protected int sampleRate = 0; - protected long numDatagrams = 0; - /** - * The total duration of the timeline data, in samples. This is only computed upon request. - */ - protected long totalDuration = -1; - - protected int datagramsBytePos = 0; - protected int timeIdxBytePos = 0; - - // exactly one of the two following variables will be non-null after load(): - private MappedByteBuffer mappedBB = null; - private FileChannel fileChannel = null; - - /****************/ - /* CONSTRUCTORS */ - /****************/ - - /** - * Construct a timeline from the given file name. - * - * Aiming for the fundamental guarantee: If an instance of this class is created, it is usable. - * - * @param fileName - * The file to read the timeline from. Must be non-null and point to a valid timeline file. - * @throws NullPointerException - * if null argument is given - * @throws MaryConfigurationException - * if no timeline reader can be instantiated from fileName - */ - public TimelineReader(String fileName) throws MaryConfigurationException { - this(fileName, true); - } - - /** - * Construct a timeline from the given file name. - * - * Aiming for the fundamental guarantee: If an instance of this class is created, it is usable. - * - * @param fileName - * The file to read the timeline from. Must be non-null and point to a valid timeline file. - * @param tryMemoryMapping - * if true, will attempt to read audio data via a memory map, and fall back to piecewise reading. If false, will - * immediately go for piecewise reading using a RandomAccessFile. - * @throws NullPointerException - * if null argument is given - * @throws MaryConfigurationException - * if no timeline reader can be instantiated from fileName - */ - public TimelineReader(String fileName, boolean tryMemoryMapping) throws MaryConfigurationException { - if (fileName == null) { - throw new NullPointerException("Filename is null"); - } - try { - load(fileName, tryMemoryMapping); - } catch (Exception e) { - throw new MaryConfigurationException("Cannot load timeline file from " + fileName, e); - } - } - - /** - * Only subclasses can instantiate a TimelineReader object that doesn't call {@link #load(String)}. It is their responsibility - * then to ensure the fundamental guarantee. - */ - protected TimelineReader() { - } - - /** - * Load a timeline from a file. - * - * @param fileName - * The file to read the timeline from. Must be non-null and point to a valid timeline file. - * - * @throws IOException - * if a problem occurs during reading - * @throws BufferUnderflowException - * if a problem occurs during reading - * @throws MaryConfigurationException - * if fileName does not point to a valid timeline file - */ - protected void load(String fileName) throws IOException, BufferUnderflowException, MaryConfigurationException, - NullPointerException { - load(fileName, true); - } - - /** - * Load a timeline from a file. - * - * @param fileName - * The file to read the timeline from. Must be non-null and point to a valid timeline file. - * - * @throws IOException - * if a problem occurs during reading - * @throws BufferUnderflowException - * if a problem occurs during reading - * @throws MaryConfigurationException - * if fileName does not point to a valid timeline file - */ - protected void load(String fileName, boolean tryMemoryMapping) throws IOException, BufferUnderflowException, - MaryConfigurationException, NullPointerException { - assert fileName != null : "filename is null"; - - RandomAccessFile file = new RandomAccessFile(fileName, "r"); - FileChannel fc = file.getChannel(); - // Expect header to be no bigger than 64k bytes - ByteBuffer headerBB = ByteBuffer.allocate(0x10000); - fc.read(headerBB); - headerBB.limit(headerBB.position()); - headerBB.position(0); - - maryHdr = new MaryHeader(headerBB); - if (maryHdr.getType() != MaryHeader.TIMELINE) { - throw new MaryConfigurationException("File is not a valid timeline file."); - } - /* Load the processing info header */ - procHdr = new ProcHeader(headerBB); - - /* Load the timeline dimensions */ - sampleRate = headerBB.getInt(); - numDatagrams = headerBB.getLong(); - if (sampleRate <= 0 || numDatagrams < 0) { - throw new MaryConfigurationException("Illegal values in timeline file."); - } - - /* Load the positions of the various subsequent components */ - datagramsBytePos = (int) headerBB.getLong(); - timeIdxBytePos = (int) headerBB.getLong(); - if (timeIdxBytePos < datagramsBytePos) { - throw new MaryConfigurationException("File seems corrupt: index is expected after data, not before"); - } - - /* Go fetch the time index at the end of the file */ - fc.position(timeIdxBytePos); - ByteBuffer indexBB = ByteBuffer.allocate((int) (fc.size() - timeIdxBytePos)); - fc.read(indexBB); - indexBB.limit(indexBB.position()); - indexBB.position(0); - idx = new Index(indexBB); - - if (tryMemoryMapping) { - // Try if we can use a mapped byte buffer: - try { - mappedBB = fc.map(FileChannel.MapMode.READ_ONLY, datagramsBytePos, timeIdxBytePos - datagramsBytePos); - file.close(); // if map() succeeded, we don't need the file anymore. - } catch (IOException ome) { - MaryUtils.getLogger("Timeline").warn( - "Cannot use memory mapping for timeline file '" + fileName + "' -- falling back to piecewise reading"); - } - } - if (!tryMemoryMapping || mappedBB == null) { // use piecewise reading - fileChannel = fc; - assert fileChannel != null; - // and leave file open - } - - // postconditions: - assert idx != null; - assert procHdr != null; - assert fileChannel == null && mappedBB != null || fileChannel != null && mappedBB == null; - } - - /** - * Return the content of the processing header as a String. - * - * @return a non-null string representing the proc header. - */ - public String getProcHeaderContents() { - return procHdr.getString(); - } - - /** - * Returns the number of datagrams in the timeline. - * - * @return the (non-negative) number of datagrams, as a long. - */ - public long getNumDatagrams() { - assert numDatagrams >= 0; - return numDatagrams; - } - - /** - * Returns the position of the datagram zone in the original file. - * - * @return the byte position of the datagram zone. - */ - protected long getDatagramsBytePos() { - return datagramsBytePos; - } - - /** - * Returns the timeline's sample rate. - * - * @return the sample rate as a positive integer. - */ - public int getSampleRate() { - assert sampleRate > 0; - return sampleRate; - } - - /** - * Return the total duration of all data in this timeline. Implementation note: this is an expensive operation that should not - * be used in production. - * - * @return a non-negative long representing the accumulated duration of all datagrams. - * @throws MaryConfigurationException - * if the duration cannot be obtained. - */ - public long getTotalDuration() throws MaryConfigurationException { - if (totalDuration == -1) { - computeTotalDuration(); - } - assert totalDuration >= 0; - return totalDuration; - } - - /** - * Compute the total duration of a timeline. This is an expensive method, since it goes through all datagrams to compute this - * duration. It should not normally be used in production. - * - * @throws MaryConfigurationException - * if the duration could not be computed. - */ - protected void computeTotalDuration() throws MaryConfigurationException { - long time = 0; - long nRead = 0; - boolean haveReadAll = false; - try { - Pair p = getByteBufferAtTime(0); - ByteBuffer bb = p.getFirst(); - assert p.getSecond() == 0; - while (!haveReadAll) { - Datagram dat = getNextDatagram(bb); - if (dat == null) { - // we may have reached the end of the current byte buffer... try reading another: - p = getByteBufferAtTime(time); - bb = p.getFirst(); - assert p.getSecond() == time; - dat = getNextDatagram(bb); - if (dat == null) { // no, indeed we cannot read any more - break; // abort, we could not read all - } - } - assert dat != null; - time += dat.getDuration(); // duration in timeline sample rate - nRead++; // number of datagrams read - if (nRead == numDatagrams) { - haveReadAll = true; - } - } - } catch (Exception e) { - throw new MaryConfigurationException("Could not compute total duration", e); - } - if (!haveReadAll) { - throw new MaryConfigurationException("Could not read all datagrams to compute total duration"); - } - totalDuration = time; - } - - /** - * The index object. - * - * @return the non-null index object. - */ - public Index getIndex() { - assert idx != null; - return idx; - } - - // Helper methods - - /** - * Scales a discrete time to the timeline's sample rate. - * - * @param reqSampleRate - * the externally given sample rate. - * @param targetTimeInSamples - * a discrete time, with respect to the externally given sample rate. - * - * @return a discrete time, in samples with respect to the timeline's sample rate. - */ - protected long scaleTime(int reqSampleRate, long targetTimeInSamples) { - if (reqSampleRate == sampleRate) - return (targetTimeInSamples); - /* else */return ((long) Math.round((double) (reqSampleRate) * (double) (targetTimeInSamples) / (double) (sampleRate))); - } - - /** - * Unscales a discrete time from the timeline's sample rate. - * - * @param reqSampleRate - * the externally given sample rate. - * @param timelineTimeInSamples - * a discrete time, with respect to the timeline sample rate. - * - * @return a discrete time, in samples with respect to the externally given sample rate. - */ - protected long unScaleTime(int reqSampleRate, long timelineTimeInSamples) { - if (reqSampleRate == sampleRate) - return (timelineTimeInSamples); - /* else */return ((long) Math.round((double) (sampleRate) * (double) (timelineTimeInSamples) / (double) (reqSampleRate))); - } - - /******************/ - /* DATA ACCESSORS */ - /******************/ - - /** - * Skip the upcoming datagram at the current position of the byte buffer. - * - * @return the duration of the datagram we skipped - * @throws IOException - * if we cannot skip another datagram because we have reached the end of the byte buffer - */ - protected long skipNextDatagram(ByteBuffer bb) throws IOException { - long datagramDuration = bb.getLong(); - int datagramSize = bb.getInt(); - if (bb.position() + datagramSize > bb.limit()) { - throw new IOException("cannot skip datagram: it is not fully contained in byte buffer"); - } - bb.position(bb.position() + datagramSize); - return datagramDuration; - } - - /** - * Read and return the upcoming datagram from the given byte buffer. Subclasses should override this method to create - * subclasses of Datagram. - * - * @param bb - * the timeline byte buffer to read from - * - * @return the current datagram, or null if EOF was encountered - */ - protected Datagram getNextDatagram(ByteBuffer bb) { - assert bb != null; - // If the end of the datagram zone is reached, refuse to read - if (bb.position() == bb.limit()) { - return null; - } - // Else, read the datagram from the file - try { - return new Datagram(bb); - } catch (IOException ioe) { - return null; - } - } - - /** - * Hop the datagrams in the given byte buffer until the one which begins at or contains the desired time (time is in samples; - * the sample rate is assumed to be that of the timeline). - * - * @param bb - * the timeline byte buffer to use. Must not be null. - * @param currentTimeInSamples - * the time position corresponding to the current position of the byte buffer. Must not be negative. - * @param targetTimeInSamples - * the time location to reach. Must not be less than currentTimeInSamples - * - * @return the actual time at which we end up after hopping. This is less than or equal to targetTimeInSamples, never greater - * than it. - * @throws IOException - * if there is a problem skipping the datagrams - * @throws IllegalArgumentException - * if targetTimeInSamples is less than currentTimeInSamples - */ - protected long hopToTime(ByteBuffer bb, long currentTimeInSamples, long targetTimeInSamples) throws IOException, - IllegalArgumentException { - assert bb != null; - assert currentTimeInSamples >= 0; - assert targetTimeInSamples >= currentTimeInSamples : "Cannot hop back from time " + currentTimeInSamples + " to time " - + targetTimeInSamples; - - /* - * If the current time position is the requested time do nothing, you are already at the right position - */ - if (currentTimeInSamples == targetTimeInSamples) { - return currentTimeInSamples; - } - /* Else hop: */ - int byteBefore = bb.position(); - long timeBefore = currentTimeInSamples; - /* Hop until the datagram which comes just after the requested time */ - while (currentTimeInSamples <= targetTimeInSamples) { // Stop after the requested time, we will step back - // to the correct time in case of equality - timeBefore = currentTimeInSamples; - byteBefore = bb.position(); - long skippedDuration = skipNextDatagram(bb); - currentTimeInSamples += skippedDuration; - } - /* Do one step back so that the pointed datagram contains the requested time */ - bb.position(byteBefore); - return timeBefore; - } - - /** - * This method produces a new byte buffer whose current position represents the requested positionInFile. It cannot be assumed - * that a call to byteBuffer.position() produces any meaningful values. The byte buffer may represent only a part of the - * available data; however, at least one datagram can be read from the byte buffer. If no further data can be read from it, a - * new byte buffer must be obtained by calling this method again with a new target time. - * - * @param targetTimeInSamples - * the time position in the file which should be accessed as a byte buffer, in samples. Must be non-negative and - * less than the total duration of the timeline. - * @return a pair representing the byte buffer from which to read, and the exact time corresponding to the current position of - * the byte buffer. The position as such is not meaningful; the time is guaranteed to be less than or equal to - * targetTimeInSamples. - * @throws IOException - * , BufferUnderflowException if no byte buffer can be obtained for the requested time. - */ - protected Pair getByteBufferAtTime(long targetTimeInSamples) throws IOException, BufferUnderflowException { - if (mappedBB != null) { - return getMappedByteBufferAtTime(targetTimeInSamples); - } else { - return loadByteBufferAtTime(targetTimeInSamples); - } - } - - protected Pair getMappedByteBufferAtTime(long targetTimeInSamples) throws IllegalArgumentException, - IOException { - assert mappedBB != null; - /* Seek for the time index which comes just before the requested time */ - IdxField idxFieldBefore = idx.getIdxFieldBefore(targetTimeInSamples); - long time = idxFieldBefore.timePtr; - int bytePos = (int) (idxFieldBefore.bytePtr - datagramsBytePos); - ByteBuffer bb = mappedBB.duplicate(); - bb.position(bytePos); - time = hopToTime(bb, time, targetTimeInSamples); - return new Pair(bb, time); - } - - protected Pair loadByteBufferAtTime(long targetTimeInSamples) throws IOException { - assert fileChannel != null; - // we must load a chunk of data from the FileChannel - int bufSize = 0x10000; // 64 kB - /* Seek for the time index which comes just before the requested time */ - IdxField idxFieldBefore = idx.getIdxFieldBefore(targetTimeInSamples); - long time = idxFieldBefore.timePtr; - long bytePos = idxFieldBefore.bytePtr; - if (bytePos + bufSize > timeIdxBytePos) { // must not read index data as datagrams - bufSize = (int) (timeIdxBytePos - bytePos); - } - ByteBuffer bb = loadByteBuffer(bytePos, bufSize); - - while (true) { - if (!canReadDatagramHeader(bb)) { - bb = loadByteBuffer(bytePos, bufSize); - assert canReadDatagramHeader(bb); - } - int posBefore = bb.position(); - Datagram d = new Datagram(bb, false); - if (time + d.getDuration() > targetTimeInSamples) { // d is our datagram - bb.position(posBefore); - int datagramNumBytes = Datagram.NUM_HEADER_BYTES + d.getLength(); - // need to make sure we return a byte buffer from which d can be read - if (!canReadAmount(bb, datagramNumBytes)) { - bb = loadByteBuffer(bytePos, Math.max(datagramNumBytes, bufSize)); - } - assert canReadAmount(bb, datagramNumBytes); - break; - } else { - // keep on skipping - time += d.getDuration(); - if (canReadAmount(bb, d.getLength())) { - bb.position(bb.position() + d.getLength()); - } else { - bytePos += bb.position(); - bytePos += d.getLength(); - bb = loadByteBuffer(bytePos, bufSize); - } - } - } - return new Pair(bb, time); - } - - /** - * @param bytePos - * position in fileChannel from which to load the byte buffer - * @param bufSize - * size of the byte buffer - * @return the byte buffer, loaded and set such that limit is bufSize and position is 0 - * @throws IOException - * if the data cannot be read from fileChannel - */ - private ByteBuffer loadByteBuffer(long bytePos, int bufSize) throws IOException { - ByteBuffer bb = ByteBuffer.allocate(bufSize); - fileChannel.read(bb, bytePos); // this will block if another thread is currently reading from fileChannel - bb.limit(bb.position()); - bb.position(0); - return bb; - } - - private boolean canReadDatagramHeader(ByteBuffer bb) { - return canReadAmount(bb, Datagram.NUM_HEADER_BYTES); - } - - private boolean canReadAmount(ByteBuffer bb, int amount) { - return bb.limit() - bb.position() >= amount; - } - - /** - * Get a single datagram from a particular time location, given in the timeline's sampling rate. - * - * @param targetTimeInSamples - * the requested position, in samples. Must be non-negative and less than the total duration of the timeline. - * - * @return the datagram starting at or overlapping the given time, or null if end-of-file was encountered - * @throws IOException - * , BufferUnderflowException if no datagram could be created from the data at the given time. - */ - public Datagram getDatagram(long targetTimeInSamples) throws IOException { - Pair p = getByteBufferAtTime(targetTimeInSamples); - ByteBuffer bb = p.getFirst(); - return getNextDatagram(bb); - } - - /** - * Get a single datagram from a particular time location. - * - * @param targetTimeInSamples - * the requested position, in samples. Must be non-negative and less than the total duration of the timeline. - * @param reqSampleRate - * the sample rate for the requested times. - * - * @return the datagram starting at or overlapping the given time, or null if end-of-file was encountered - * @throws IOException - * if no datagram could be created from the data at the given time. - */ - public Datagram getDatagram(long targetTimeInSamples, int reqSampleRate) throws IOException { - /* - * Resample the requested time location, in case the sample times are different between the request and the timeline - */ - long scaledTargetTime = scaleTime(reqSampleRate, targetTimeInSamples); - Datagram dat = getDatagram(scaledTargetTime); - if (dat == null) - return null; - if (reqSampleRate != sampleRate) - dat.setDuration(unScaleTime(reqSampleRate, dat.getDuration())); // => Don't forget to stay time-consistent! - return dat; - } - - /** - * Get the datagrams spanning a particular time range from a particular time location, and return the time offset between the - * time request and the actual location of the first returned datagram. Irrespective of the values of nDatagrams and - * timeSpanInSamples, at least one datagram is always returned. - * - * @param targetTimeInSamples - * the requested position, in samples. Must be non-negative and less than the total duration of the timeline. - * @param nDatagrams - * the number of datagrams to read. Ignored if timeSpanInSamples is positive. - * @param timeSpanInSamples - * the requested time span, in samples. If positive, then datagrams are selected by the given time span. - * @param reqSampleRate - * the sample rate for the requested and returned times. Must be positive. - * @param returnOffset - * an optional output field. If it is not null, then after the call it must have length of at least 1, and the - * first array field will contain the time difference, in samples, between the time request and the actual - * beginning of the first datagram. - * - * @return an array of datagrams containing at least one datagram. If less than the requested amount of datagrams can be read, - * the number of datagrams that can be read is returned. - * @throws IllegalArgumentException - * if targetTimeInSamples is negative, or if a returnOffset of length 0 is given. - * @throws IOException - * if no data can be read at the given target time - */ - private Datagram[] getDatagrams(long targetTimeInSamples, int nDatagrams, long timeSpanInSamples, int reqSampleRate, - long[] returnOffset) throws IllegalArgumentException, IOException { - /* Check the input arguments */ - if (targetTimeInSamples < 0) { - throw new IllegalArgumentException("Can't get a datagram from a negative time position (given time position was [" - + targetTimeInSamples + "])."); - } - if (reqSampleRate <= 0) { - throw new IllegalArgumentException("sample rate must be positive, but is " + reqSampleRate); - } - // Get the datagrams by number or by time span? - boolean byNumber; - if (timeSpanInSamples > 0) { - byNumber = false; - } else { - byNumber = true; - if (nDatagrams <= 0) { - nDatagrams = 1; // return at least one datagram - } - } - - /* - * Resample the requested time location, in case the sample times are different between the request and the timeline - */ - long scaledTargetTime = scaleTime(reqSampleRate, targetTimeInSamples); - - Pair p = getByteBufferAtTime(scaledTargetTime); - ByteBuffer bb = p.getFirst(); - long time = p.getSecond(); - if (returnOffset != null) { // return offset between target and actual start time - if (returnOffset.length == 0) { - throw new IllegalArgumentException("If returnOffset is given, it must have length of at least 1"); - } - returnOffset[0] = unScaleTime(reqSampleRate, (scaledTargetTime - time)); - } - - ArrayList datagrams = new ArrayList(byNumber ? nDatagrams : 10); - // endTime is stop criterion if reading by time scale: - long endTime = byNumber ? -1 : scaleTime(reqSampleRate, (targetTimeInSamples + timeSpanInSamples)); - int nRead = 0; - boolean haveReadAll = false; - while (!haveReadAll) { - Datagram dat = getNextDatagram(bb); - if (dat == null) { - // we may have reached the end of the current byte buffer... try reading another: - try { - p = getByteBufferAtTime(time); - } catch (Exception ioe) { - // cannot get another byte buffer -- stop reading. - break; - } - bb = p.getFirst(); - dat = getNextDatagram(bb); - if (dat == null) { // no, indeed we cannot read any more - break; // abort, we could not read all - } - } - assert dat != null; - time += dat.getDuration(); // duration in timeline sample rate - nRead++; // number of datagrams read - if (reqSampleRate != sampleRate) { - dat.setDuration(unScaleTime(reqSampleRate, dat.getDuration())); // convert duration into reqSampleRate - } - datagrams.add(dat); - if (byNumber && nRead == nDatagrams || !byNumber && time >= endTime) { - haveReadAll = true; - } - } - return (Datagram[]) datagrams.toArray(new Datagram[0]); - } - - // ///////////////////// Convenience methods: variants of getDatagrams() /////////////////////// - - // ///////////////////// by time span //////////////////////////// - - /** - * Get the datagrams spanning a particular time range from a particular time location, and return the time offset between the - * time request and the actual location of the first returned datagram. Irrespective of the value of timeSpanInSamples, at - * least one datagram is always returned. - * - * @param targetTimeInSamples - * the requested position, in samples. Must be non-negative and less than the total duration of the timeline. - * @param timeSpanInSamples - * the requested time span, in samples. If positive, then datagrams are selected by the given time span. - * @param reqSampleRate - * the sample rate for the requested and returned times. Must be positive. - * @param returnOffset - * an optional output field. If it is not null, then after the call it must have length of at least 1, and the - * first array field will contain the time difference, in samples, between the time request and the actual - * beginning of the first datagram. - * - * @return an array of datagrams containing at least one datagram. If less than the requested amount of datagrams can be read, - * the number of datagrams that can be read is returned. - * @throws IllegalArgumentException - * if targetTimeInSamples is negative, or if a returnOffset of length 0 is given. - * @throws IOException - * , BufferUnderflowException if no data can be read at the given target time - */ - public Datagram[] getDatagrams(long targetTimeInSamples, long timeSpanInSamples, int reqSampleRate, long[] returnOffset) - throws IOException { - return getDatagrams(targetTimeInSamples, -1, timeSpanInSamples, reqSampleRate, returnOffset); - } - - /** - * Get the datagrams spanning a particular time range from a particular time location. Irrespective of the value of - * timeSpanInSamples, at least one datagram is always returned. - * - * @param targetTimeInSamples - * the requested position, in samples. Must be non-negative and less than the total duration of the timeline. - * @param timeSpanInSamples - * the requested time span, in samples. If positive, then datagrams are selected by the given time span. - * @param reqSampleRate - * the sample rate for the requested and returned times. Must be positive. - * - * @return an array of datagrams containing at least one datagram. If less than the requested amount of datagrams can be read, - * the number of datagrams that can be read is returned. - * @throws IllegalArgumentException - * if targetTimeInSamples is negative, or if a returnOffset of length 0 is given. - * @throws IOException - * if no data can be read at the given target time - */ - public Datagram[] getDatagrams(long targetTimeInSamples, long timeSpanInSamples, int reqSampleRate) throws IOException { - return getDatagrams(targetTimeInSamples, timeSpanInSamples, reqSampleRate, null); - } - - /** - * Get a given number of datagrams from a particular time location. - * - * @param targetTimeInSamples - * the requested position, in samples. Must be non-negative and less than the total duration of the timeline. - * @param number - * the number of datagrams to read. Even if this is <= 0, at least one datagram is always returned. - * @param reqSampleRate - * the sample rate for the requested and returned times. Must be positive. - * - * @return an array of datagrams containing at least one datagram. If less than the requested amount of datagrams can be read, - * the number of datagrams that can be read is returned. - * @throws IllegalArgumentException - * if targetTimeInSamples is negative, or if a returnOffset of length 0 is given. - * @throws IOException - * if no data can be read at the given target time - */ - public Datagram[] getDatagrams(long targetTimeInSamples, long timeSpanInSamples) throws IOException { - return getDatagrams(targetTimeInSamples, timeSpanInSamples, sampleRate, null); - } - - // ///////////////////// by number of datagrams //////////////////////////// - - /** - * Get a given number of datagrams from a particular time location, and return the time offset between the time request and - * the actual location of the first returned datagram. - * - * @param targetTimeInSamples - * the requested position, in samples. Must be non-negative and less than the total duration of the timeline. - * @param number - * the number of datagrams to read. Even if this is <= 0, at least one datagram is always returned. - * @param reqSampleRate - * the sample rate for the requested and returned times. Must be positive. - * @param returnOffset - * an optional output field. If it is not null, then after the call it must have length of at least 1, and the - * first array field will contain the time difference, in samples, between the time request and the actual - * beginning of the first datagram. - * - * @return an array of datagrams containing at least one datagram. If less than the requested amount of datagrams can be read, - * the number of datagrams that can be read is returned. - * @throws IllegalArgumentException - * if targetTimeInSamples is negative, or if a returnOffset of length 0 is given. - * @throws IOException - * if no data can be read at the given target time - */ - public Datagram[] getDatagrams(long targetTimeInSamples, int number, int reqSampleRate, long[] returnOffset) - throws IOException { - return getDatagrams(targetTimeInSamples, number, -1, reqSampleRate, returnOffset); - } - - // ///////////////////// by unit //////////////////////////// - - /** - * Get the datagrams spanning a particular unit, and return the time offset between the unit request and the actual location - * of the first returned datagram. Irrespective of the unit duration, at least one datagram is always returned. - * - * @param unit - * The requested speech unit, containing its own position and duration. - * @param reqSampleRate - * the sample rate for the requested and returned times. Must be positive. - * @param returnOffset - * an optional output field. If it is not null, then after the call it must have length of at least 1, and the - * first array field will contain the time difference, in samples, between the time request and the actual - * beginning of the first datagram. - * - * @return an array of datagrams containing at least one datagram. If less than the requested amount of datagrams can be read, - * the number of datagrams that can be read is returned. - * @throws IllegalArgumentException - * if targetTimeInSamples is negative, or if a returnOffset of length 0 is given. - * @throws IOException - * if no data can be read at the given target time - */ - public Datagram[] getDatagrams(Unit unit, int reqSampleRate, long[] returnOffset) throws IOException { - return getDatagrams(unit.startTime, (long) (unit.duration), reqSampleRate, returnOffset); - } - - /** - * Get the datagrams spanning a particular unit. Irrespective of the unit duration, at least one datagram is always returned. - * - * @param unit - * The requested speech unit, containing its own position and duration. - * @param reqSampleRate - * the sample rate for the requested and returned times. Must be positive. - * - * @return an array of datagrams containing at least one datagram. If less than the requested amount of datagrams can be read, - * the number of datagrams that can be read is returned. - * @throws IllegalArgumentException - * if targetTimeInSamples is negative, or if a returnOffset of length 0 is given. - * @throws IOException - * if no data can be read at the given target time - */ - public Datagram[] getDatagrams(Unit unit, int reqSampleRate) throws IOException { - return getDatagrams(unit, reqSampleRate, null); - } - - /*****************************************/ - /* HELPER CLASSES */ - /*****************************************/ - - /** - * Simple helper class to read the index part of a timeline file. The index points to datagrams at or before a certain point - * in time. - * - * Note: If no datagram starts at the exact index time, it makes sense to point to the previous datagram rather than the - * following one. - * - * If one would store the location of the datagram which comes just after the index position (the currently tested datagram), - * there would be a possibility that a particular time request falls between the index and the datagram: - * - * time axis ---------------------------------> INDEX <-- REQUEST | ---------------> DATAGRAM - * - * This would require a subsequent backwards time hopping, which is impossible because the datagrams are a singly linked list. - * - * By registering the location of the previous datagram, any time request will find an index which points to a datagram - * falling BEFORE or ON the index location: - * - * time axis ---------------------------------> INDEX <-- REQUEST | DATAGRAM <--- - * - * Thus, forward hopping is always possible and the requested time can always be reached. - * - * @author sacha - */ - public static class Index { - private int idxInterval = 0; // The fixed time interval (in samples) separating two index fields. - - /** - * For index field i, bytePtrs[i] is the position in bytes, from the beginning of the file, of the datagram coming on or - * just before that index field. - */ - private long[] bytePtrs; - - /** - * For index field i, timePtrs[i] is the time position in samples of the datagram coming on or just before that index - * field. - */ - private long[] timePtrs; - - /****************/ - /* CONSTRUCTORS */ - /****************/ - - /** - * Construct an index from a data input stream or random access file. Fundamental guarantee: Once created, the index is - * guaranteed to contain a positive index interval and monotonously rising byte and time pointers. - * - * @param bb - * byte buffer from which to read the index. Must not be null, and read position must be at start of index. - * @throws IOException - * if there is a problem reading. - * @throws MaryConfigurationException - * if the index is not well-formed. - */ - private Index(DataInput raf) throws IOException, MaryConfigurationException { - assert raf != null : "null argument"; - load(raf); - } - - /** - * Construct an index from a byte buffer. Fundamental guarantee: Once created, the index is guaranteed to contain a - * positive index interval and monotonously rising byte and time pointers. - * - * @param rafIn - * data input from which to read the index. Must not be null, and read position must be at start of index. - * @throws BufferUnderflowException - * if there is a problem reading. - * @throws MaryConfigurationException - * if the index is not well-formed. - */ - private Index(ByteBuffer bb) throws BufferUnderflowException, MaryConfigurationException { - assert bb != null : "null argument"; - load(bb); - } - - /** - * Constructor which builds a new index with a specific index interval and a given sample rate. Fundamental guarantee: - * Once created, the index is guaranteed to contain a positive index interval and monotonously rising byte and time - * pointers. - * - * @param idxInterval - * the index interval, in samples. Must be a positive number. - * @param indexFields - * the actual index data. Must not be null. - * @throws IllegalArgumentException - * if the index data given is not well-formed. - * @throws NullPointerException - * if indexFields are null. - */ - public Index(int idxInterval, Vector indexFields) throws IllegalArgumentException, NullPointerException { - if (idxInterval <= 0) { - throw new IllegalArgumentException("got index interval <= 0"); - } - if (indexFields == null) { - throw new NullPointerException("null argument"); - } - this.idxInterval = idxInterval; - bytePtrs = new long[indexFields.size()]; - timePtrs = new long[indexFields.size()]; - for (int i = 0; i < bytePtrs.length; i++) { - IdxField f = indexFields.get(i); - bytePtrs[i] = f.bytePtr; - timePtrs[i] = f.timePtr; - if (i > 0) { - if (bytePtrs[i] < bytePtrs[i - 1] || timePtrs[i] < timePtrs[i - 1]) { - throw new IllegalArgumentException( - "Pointer positions in index fields must be strictly monotonously rising"); - } - } - } - } - - /*****************/ - /* I/O METHODS */ - /*****************/ - - /** - * Method which loads an index from a data input (random access file or data input stream). - * - * @param rafIn - * data input from which to read the index. Must not be null, and read position must be at start of index. - * @throws IOException - * if there is a problem reading. - * @throws MaryConfigurationException - * if the index is not well-formed. - */ - public void load(DataInput rafIn) throws IOException, MaryConfigurationException { - int numIdx = rafIn.readInt(); - idxInterval = rafIn.readInt(); - if (idxInterval <= 0) { - throw new MaryConfigurationException("read negative index interval -- file seems corrupt"); - } - - bytePtrs = new long[numIdx]; - timePtrs = new long[numIdx]; - int numBytesToRead = 16 * numIdx + 16; // 2*8 bytes for each index field + 16 for prevBytePos and prevTimePos - - byte[] data = new byte[numBytesToRead]; - rafIn.readFully(data); - DataInput bufIn = new DataInputStream(new ByteArrayInputStream(data)); - - for (int i = 0; i < numIdx; i++) { - bytePtrs[i] = bufIn.readLong(); - timePtrs[i] = bufIn.readLong(); - if (i > 0) { - if (bytePtrs[i] < bytePtrs[i - 1] || timePtrs[i] < timePtrs[i - 1]) { - throw new MaryConfigurationException( - "File seems corrupt: Pointer positions in index fields are not strictly monotonously rising"); - } - } - } - /* Obsolete: Read the "last datagram" memory */ - /* prevBytePos = */bufIn.readLong(); - /* prevTimePos = */bufIn.readLong(); - } - - /** - * Method which loads an index from a byte buffer. - * - * @param bb - * byte buffer from which to read the index. Must not be null, and read position must be at start of index. - * @throws BufferUnderflowException - * if there is a problem reading. - * @throws MaryConfigurationException - * if the index is not well-formed. - */ - private void load(ByteBuffer bb) throws BufferUnderflowException, MaryConfigurationException { - int numIdx = bb.getInt(); - idxInterval = bb.getInt(); - if (idxInterval <= 0) { - throw new MaryConfigurationException("read negative index interval -- file seems corrupt"); - } - - bytePtrs = new long[numIdx]; - timePtrs = new long[numIdx]; - - for (int i = 0; i < numIdx; i++) { - bytePtrs[i] = bb.getLong(); - timePtrs[i] = bb.getLong(); - if (i > 0) { - if (bytePtrs[i] < bytePtrs[i - 1] || timePtrs[i] < timePtrs[i - 1]) { - throw new MaryConfigurationException( - "File seems corrupt: Pointer positions in index fields are not strictly monotonously rising"); - } - } - } - /* Obsolete: Read the "last datagram" memory */ - /* prevBytePos = */bb.getLong(); - /* prevTimePos = */bb.getLong(); - } - - /** - * Method which writes an index to a RandomAccessFile - * */ - public long dump(RandomAccessFile rafIn) throws IOException { - long nBytes = 0; - int numIdx = getNumIdx(); - rafIn.writeInt(numIdx); - nBytes += 4; - rafIn.writeInt(idxInterval); - nBytes += 4; - for (int i = 0; i < numIdx; i++) { - rafIn.writeLong(bytePtrs[i]); - nBytes += 8; - rafIn.writeLong(timePtrs[i]); - nBytes += 8; - } - // Obsolete, keep only for file format compatibility: - // Register the "last datagram" memory as an additional field - // rafIn.writeLong(prevBytePos); - // rafIn.writeLong(prevTimePos); - rafIn.writeLong(0l); - rafIn.writeLong(0l); - nBytes += 16l; - - return nBytes; - } - - /** - * Method which writes an index to stdout - * */ - public void print() { - System.out.println(""); - int numIdx = getNumIdx(); - System.out.println("interval = " + idxInterval); - System.out.println("numIdx = " + numIdx); - for (int i = 0; i < numIdx; i++) { - System.out.println("( " + bytePtrs[i] + " , " + timePtrs[i] + " )"); - } - /* Obsolete: Register the "last datagram" memory as an additional field */ - // System.out.println( "Last datagram: " - // + "( " + prevBytePos + " , " + prevTimePos + " )" ); - System.out.println(""); - } - - /*****************/ - /* ACCESSORS */ - /*****************/ - /** - * The number of index entries. - */ - public int getNumIdx() { - return bytePtrs.length; - } - - /** - * The interval, in samples, between two index entries. - * - * @return - */ - public int getIdxInterval() { - return idxInterval; - } - - public IdxField getIdxField(int i) { - if (i < 0) { - throw new IndexOutOfBoundsException("Negative index."); - } - if (i >= bytePtrs.length) { - throw new IndexOutOfBoundsException("Requested index no. " + i + ", but highest is " + bytePtrs.length); - } - return new IdxField(bytePtrs[i], timePtrs[i]); - } - - /*****************/ - /* OTHER METHODS */ - /*****************/ - - /** - * Returns the index field that comes immediately before or straight on the requested time. - * - * @param timePosition - * the non-negative time - * @return an index field representing the index position just before or straight on the requested time. - * @throws IllegalArgumentException - * if the given timePosition is negtive - */ - public IdxField getIdxFieldBefore(long timePosition) { - if (timePosition < 0) { - throw new IllegalArgumentException("Negative time given"); - } - int index = (int) (timePosition / idxInterval); /* - * <= This is an integer division between two longs, implying a - * flooring operation on the decimal result. - */ - // System.out.println( "TIMEPOS=" + timePosition + " IDXINT=" + idxInterval + " IDX=" + idx ); - // System.out.flush(); - if (index < 0) { - throw new RuntimeException("Negative index field: [" + index + "] encountered when getting index before time=[" - + timePosition + "] (idxInterval=[" + idxInterval + "])."); - } - if (index >= bytePtrs.length) { - index = bytePtrs.length - 1; // <= Protection against ArrayIndexOutOfBounds exception due to "time out of bounds" - } - return new IdxField(bytePtrs[index], timePtrs[index]); - } - } - - /** - * Simple helper class to read the index fields in a timeline. - * - * @author sacha - * - */ - public static class IdxField { - // TODO: rethink if these should be public fields or if we should add accessors. - public long bytePtr = 0; - public long timePtr = 0; - - public IdxField() { - bytePtr = 0; - timePtr = 0; - } - - public IdxField(long setBytePtr, long setTimePtr) { - bytePtr = setBytePtr; - timePtr = setTimePtr; - } - } - - /** - * - * Simple helper class to load the processing header. - * - * @author sacha - * - */ - public static class ProcHeader { - - private String procHeader = null; - - /****************/ - /* CONSTRUCTORS */ - /****************/ - - /** - * Constructor which loads the procHeader from a RandomAccessFile. Fundamental guarantee: after creation, the ProcHeader - * object has a non-null (but possibly empty) string content. - * - * @param raf - * input from which to load the processing header. Must not be null and must be positioned so that a processing - * header can be read from it. - * - * @throws IOException - * if no proc header can be read at the current position. - */ - private ProcHeader(RandomAccessFile raf) throws IOException { - loadProcHeader(raf); - } - - /** - * Constructor which loads the procHeader from a RandomAccessFile Fundamental guarantee: after creation, the ProcHeader - * object has a non-null (but possibly empty) string content. - * - * @param raf - * input from which to load the processing header. Must not be null and must be positioned so that a processing - * header can be read from it. - * - * @throws BufferUnderflowException - * , UTFDataFormatException if no proc header can be read at the current position. - */ - private ProcHeader(ByteBuffer bb) throws BufferUnderflowException, UTFDataFormatException { - loadProcHeader(bb); - } - - /** - * Constructor which makes the procHeader from a String. Fundamental guarantee: after creation, the ProcHeader object has - * a non-null (but possibly empty) string content. - * - * @param procStr - * a non-null string representing the contents of the ProcHeader. - * @throws NullPointerException - * if procStr is null - * */ - public ProcHeader(String procStr) { - if (procStr == null) { - throw new NullPointerException("null argument"); - } - procHeader = procStr; - } - - /****************/ - /* ACCESSORS */ - /****************/ - - /** - * Return the string length of the proc header. - * - * @return a non-negative int representling the string length of the proc header. - */ - public int getCharSize() { - assert procHeader != null; - return procHeader.length(); - } - - /** - * Get the string content of the proc header. - * - * @return a non-null string representing the string content of the proc header. - */ - public String getString() { - assert procHeader != null; - return procHeader; - } - - /*****************/ - /* I/O METHODS */ - /*****************/ - - /** - * Method which loads the header from a RandomAccessFile. - * - * @param rafIn - * file to read from, must not be null. - * @throws IOException - * if no proc header can be read at the current position. - */ - private void loadProcHeader(RandomAccessFile rafIn) throws IOException { - assert rafIn != null : "null argument"; - procHeader = rafIn.readUTF(); - assert procHeader != null; - } - - /** - * Method which loads the header from a byte buffer. - * - * @param bb - * byte buffer to read from, must not be null. - * @throws BufferUnderflowException - * , UTFDataFormatException if no proc header can be read at the current position. - */ - private void loadProcHeader(ByteBuffer bb) throws BufferUnderflowException, UTFDataFormatException { - procHeader = StreamUtils.readUTF(bb); - assert procHeader != null; - } - - /** - * Method which writes the proc header to a RandomAccessFile. - * - * @return the number of written bytes. - * */ - public long dump(RandomAccessFile rafIn) throws IOException { - long before = rafIn.getFilePointer(); - rafIn.writeUTF(procHeader); - long after = rafIn.getFilePointer(); - return after - before; - } - } - -} \ No newline at end of file diff --git a/marytts-runtime/src/main/java/marytts/unitselection/data/Unit.java b/marytts-runtime/src/main/java/marytts/unitselection/data/Unit.java deleted file mode 100644 index 64c7dae6b3..0000000000 --- a/marytts-runtime/src/main/java/marytts/unitselection/data/Unit.java +++ /dev/null @@ -1,85 +0,0 @@ -/** - * Copyright 2006 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.unitselection.data; - -/** - * Representation of a unit from a unit database. This gives access to everything that is known about a given unit, including all - * sorts of features and the actual audio data. - * - * @author Marc Schröder - * - */ -public class Unit { - - /** - * Unit start time, expressed in samples. To convert into time, divide by UnitFileReader.getSampleRate(). - */ - public final long startTime; - - /** - * Unit duration, expressed in samples. To convert into time, divide by UnitFileReader.getSampleRate(). - */ - public final int duration; - - /** - * Index position of this unit in the unit file. - */ - public final int index; - - public Unit(long startTime, int duration, int index) { - this.startTime = startTime; - this.duration = duration; - this.index = index; - } - - /** - * Determine whether the unit is an "edge" unit, i.e. a unit marking the start or the end of an utterance. - * - * @param i - * The index of the considered unit. - * @return true if the unit is an edge unit, false otherwise - */ - public boolean isEdgeUnit() { - return duration == -1; - } - - public String toString() { - return "unit " + index + " start: " + startTime + ", duration: " + duration; - } - - /** - * inspired by http://www.artima.com/lejava/articles/equality.html - */ - @Override - public boolean equals(Object other) { - boolean result = false; - if (other instanceof Unit) { - Unit that = (Unit) other; - result = (this.index == that.index); - } - return result; - } - - @Override - public int hashCode() { - return this.index; - } - -} diff --git a/marytts-runtime/src/main/java/marytts/unitselection/data/UnitDatabase.java b/marytts-runtime/src/main/java/marytts/unitselection/data/UnitDatabase.java deleted file mode 100644 index b9799016bb..0000000000 --- a/marytts-runtime/src/main/java/marytts/unitselection/data/UnitDatabase.java +++ /dev/null @@ -1,195 +0,0 @@ -/** - * Copyright 2006 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.unitselection.data; - -import java.util.ArrayList; -import java.util.List; - -import marytts.cart.CART; -import marytts.unitselection.select.JoinCostFunction; -import marytts.unitselection.select.StatisticalCostFunction; -import marytts.unitselection.select.Target; -import marytts.unitselection.select.TargetCostFunction; -import marytts.unitselection.select.viterbi.ViterbiCandidate; -import marytts.util.MaryUtils; -import marytts.util.data.Datagram; -import marytts.util.dom.DomUtils; - -import org.apache.log4j.Logger; -import org.w3c.dom.Element; - -/** - * The unit database of a voice - * - * @author Marc Schröder - * - */ -public class UnitDatabase { - protected TargetCostFunction targetCostFunction; - protected JoinCostFunction joinCostFunction; - protected StatisticalCostFunction sCostFunction = null; - protected UnitFileReader unitReader; - protected int numUnits; - protected CART preselectionCART; - protected TimelineReader audioTimeline; - protected TimelineReader basenameTimeline; - protected int backtrace; - protected Logger logger = MaryUtils.getLogger("UnitDatabase"); - - public UnitDatabase() { - } - - public void load(TargetCostFunction aTargetCostFunction, JoinCostFunction aJoinCostFunction, UnitFileReader aUnitReader, - CART aPreselectionCART, TimelineReader anAudioTimeline, TimelineReader aBasenameTimeline, int backtraceLeafSize) { - this.targetCostFunction = aTargetCostFunction; - this.joinCostFunction = aJoinCostFunction; - this.unitReader = aUnitReader; - this.numUnits = (unitReader != null ? unitReader.getNumberOfUnits() : 0); - this.preselectionCART = aPreselectionCART; - this.audioTimeline = anAudioTimeline; - this.basenameTimeline = aBasenameTimeline; - this.backtrace = backtraceLeafSize; - } - - public void load(TargetCostFunction aTargetCostFunction, JoinCostFunction aJoinCostFunction, - StatisticalCostFunction asCostFunction, UnitFileReader aUnitReader, CART aPreselectionCART, - TimelineReader anAudioTimeline, TimelineReader aBasenameTimeline, int backtraceLeafSize) { - this.targetCostFunction = aTargetCostFunction; - this.joinCostFunction = aJoinCostFunction; - this.sCostFunction = asCostFunction; - this.unitReader = aUnitReader; - this.numUnits = (unitReader != null ? unitReader.getNumberOfUnits() : 0); - this.preselectionCART = aPreselectionCART; - this.audioTimeline = anAudioTimeline; - this.basenameTimeline = aBasenameTimeline; - this.backtrace = backtraceLeafSize; - } - - public TargetCostFunction getTargetCostFunction() { - return targetCostFunction; - } - - public JoinCostFunction getJoinCostFunction() { - return joinCostFunction; - } - - public UnitFileReader getUnitFileReader() { - return unitReader; - } - - public TimelineReader getAudioTimeline() { - return audioTimeline; - } - - public StatisticalCostFunction getSCostFunction() { - return sCostFunction; - } - - /** - * Preselect a set of candidates that could be used to realise the given target. - * - * @param target - * a Target object representing an optimal unit - * @return an unsorted ArrayList of ViterbiCandidates, each containing the (same) target and a - * (different) Unit object - */ - public List getCandidates(Target target) { - // BEGIN blacklisting - // The point of this is to get the value of the "blacklist" attribute in the first child element of the MaryXML - // and store it in the blacklist String variable. - // This code seems rather inelegant; perhaps there is a better way to access the MaryXML from this method? - String blacklist = ""; - String unitBasename = "This must never be null or the empty string!"; // otherwise candidate selection fails! - Element targetElement = target.getMaryxmlElement(); - blacklist = DomUtils.getAttributeFromClosestAncestorOfAnyKind(targetElement, "blacklist"); - // END blacklisting - - // logger.debug("Looking for candidates in cart "+target.getName()); - // get the cart tree and extract the candidates - int[] clist = (int[]) preselectionCART.interpret(target, backtrace); - logger.debug("For target " + target + ", selected " + clist.length + " units"); - - // Now, clist is an array of unit indexes. - List candidates = new ArrayList(); - for (int i = 0; i < clist.length; i++) { - // The target is the same for all these candidates in the queue - // remember the actual unit: - Unit unit = unitReader.getUnit(clist[i]); - candidates.add(new ViterbiCandidate(target, unit, targetCostFunction)); - } - - // Blacklisting without crazy performance drop: - // just remove candidates again if their basenames are blacklisted - java.util.Iterator candIt = candidates.iterator(); - while (candIt.hasNext()) { - ViterbiCandidate candidate = candIt.next(); - unitBasename = getFilename(candidate.getUnit()); - if (blacklist.contains(unitBasename)) { - candIt.remove(); - } - } - - return candidates; - } - - /** - * For debugging, return the basename of the original audio file from which the unit is coming, as well as the start time in - * that file. - * - * @param unit - * @return a String containing basename followed by a space and the unit's start time, in seconds, from the beginning of the - * file. If no basenameTimeline was specified for this voice, returns the string "unknown origin". - */ - public String getFilenameAndTime(Unit unit) { - if (basenameTimeline == null) - return "unknown origin"; - long[] offset = new long[1]; - try { - Datagram[] datagrams = basenameTimeline.getDatagrams(unit.startTime, 1, unitReader.getSampleRate(), offset); - Datagram filenameData = datagrams[0]; - float time = (float) offset[0] / basenameTimeline.getSampleRate(); - String filename = new String(filenameData.getData(), "UTF-8"); - return filename + " " + time; - } catch (Exception e) { - logger.warn("Problem getting filename and time for unit " + unit.index + " at time " + unit.startTime, e); - return "unknown origin"; - } - } - - /** - * For debugging, return the basename of the original audio file from which the unit is coming. - * - * @param unit - * @return a String containing basename. If no basenameTimeline was specified for this voice, returns the string - * "unknown origin". - */ - public String getFilename(Unit unit) { - // if (basenameTimeline == null) return "unknown origin"; - try { - Datagram filenameData = basenameTimeline.getDatagram(unit.startTime); - String filename = new String(filenameData.getData(), "UTF-8"); - return filename; - } catch (Exception e) { - logger.warn("Problem getting filename for unit " + unit.index, e); - return "unknown origin"; - } - } - -} diff --git a/marytts-runtime/src/main/java/marytts/unitselection/data/UnitFileReader.java b/marytts-runtime/src/main/java/marytts/unitselection/data/UnitFileReader.java deleted file mode 100644 index 4f8a9b695e..0000000000 --- a/marytts-runtime/src/main/java/marytts/unitselection/data/UnitFileReader.java +++ /dev/null @@ -1,200 +0,0 @@ -/** - * Portions Copyright 2006 DFKI GmbH. - * Portions Copyright 2001 Sun Microsystems, Inc. - * Portions Copyright 1999-2001 Language Technologies Institute, - * Carnegie Mellon University. - * All Rights Reserved. Use is subject to license terms. - * - * Permission is hereby granted, free of charge, to use and distribute - * this software and its documentation without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of this work, and to - * permit persons to whom this work is furnished to do so, subject to - * the following conditions: - * - * 1. The code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * 2. Any modifications must be clearly marked as such. - * 3. Original authors' names are not deleted. - * 4. The authors' names are not used to endorse or promote products - * derived from this software without specific prior written - * permission. - * - * DFKI GMBH AND THE CONTRIBUTORS TO THIS WORK DISCLAIM ALL WARRANTIES WITH - * REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL DFKI GMBH NOR THE - * CONTRIBUTORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL - * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR - * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS - * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF - * THIS SOFTWARE. - */ -package marytts.unitselection.data; - -import java.io.BufferedInputStream; -import java.io.DataInputStream; -import java.io.FileInputStream; -import java.io.IOException; - -import marytts.exceptions.MaryConfigurationException; -import marytts.util.data.MaryHeader; - -/** - * Loads a unit file in memory and provides accessors to the start times and durations. - * - * @author sacha - * - */ -public class UnitFileReader { - - private MaryHeader hdr = null; - private int numberOfUnits = 0; - private int sampleRate = 0; - Unit[] units; // this has visibility "default" rather than private so that other classes in the same package can access it - // directly, for efficiency reasons - - /****************/ - /* CONSTRUCTORS */ - /****************/ - - /** - * Empty constructor; need to call load() separately. - * - * @see #load(String) - */ - public UnitFileReader() { - } - - /** - * Create a unit file reader from the given unit file - * - * @param fileName - * the unit file to read - * @throws IOException - * if a problem occurs while reading - */ - public UnitFileReader(String fileName) throws IOException, MaryConfigurationException { - load(fileName); - } - - /** - * Load the given unit file - * - * @param fileName - * the unit file to read - * @throws IOException - * if a problem occurs while reading - */ - public void load(String fileName) throws IOException, MaryConfigurationException { - /* Open the file */ - DataInputStream dis = new DataInputStream(new BufferedInputStream(new FileInputStream(fileName))); - /* Load the Mary header */ - hdr = new MaryHeader(dis); - if (hdr.getType() != MaryHeader.UNITS) { - throw new MaryConfigurationException("File [" + fileName + "] is not a valid Mary Units file."); - } - /* Read the number of units */ - numberOfUnits = dis.readInt(); - if (numberOfUnits < 0) { - throw new MaryConfigurationException("File [" + fileName + "] has a negative number of units. Aborting."); - } - /* Read the sample rate */ - sampleRate = dis.readInt(); - if (sampleRate < 0) { - throw new MaryConfigurationException("File [" + fileName + "] has a negative number sample rate. Aborting."); - } - units = new Unit[numberOfUnits]; - /* Read the start times and durations */ - for (int i = 0; i < numberOfUnits; i++) { - long startTime = dis.readLong(); - int duration = dis.readInt(); - units[i] = new Unit(startTime, duration, i); - } - } - - /*****************/ - /* OTHER METHODS */ - /*****************/ - - /** - * Get the number of units in the file. - * - * @return The number of units. - */ - public int getNumberOfUnits() { - return (numberOfUnits); - } - - /** - * Get the sample rate of the file. - * - * @return The sample rate, in Hz. - */ - public int getSampleRate() { - return (sampleRate); - } - - /** - * Return the unit number i. - * - * @param i - * The index of the considered unit. - * @return The considered unit. - */ - public Unit getUnit(int i) { - return units[i]; - } - - /** - * Return an array of units from their indexes. - * - * @param i - * The indexes of the considered units. - * @return The array of considered units. - */ - public Unit[] getUnit(int[] i) { - Unit[] ret = new Unit[i.length]; - for (int k = 0; k < i.length; k++) { - ret[k] = getUnit(i[k]); - } - return (ret); - } - - /** - * Return the unit following the given unit in the original database. - * - * @param u - * a unit - * @return the next unit in the database, or null if there is no such unit. - */ - public Unit getNextUnit(Unit u) { - if (u == null || u.index >= units.length - 1 || u.index < 0) - return null; - return units[u.index + 1]; - } - - /** - * Return the unit preceding the given unit in the original database. - * - * @param u - * a unit - * @return the previous unit in the database, or null if there is no such unit. - */ - public Unit getPreviousUnit(Unit u) { - if (u == null || u.index >= units.length || u.index <= 0) - return null; - return units[u.index - 1]; - } - - /** - * Determine whether the unit number i is an "edge" unit, i.e. a unit marking the start or the end of an utterance. - * - * @param i - * The index of the considered unit. - * @return true if the unit is an edge unit in the unit file, false otherwise - */ - public boolean isEdgeUnit(int i) { - return units[i].isEdgeUnit(); - } - -} diff --git a/marytts-runtime/src/main/java/marytts/unitselection/interpolation/InterpolatingSynthesizer.java b/marytts-runtime/src/main/java/marytts/unitselection/interpolation/InterpolatingSynthesizer.java deleted file mode 100644 index 0d035f4342..0000000000 --- a/marytts-runtime/src/main/java/marytts/unitselection/interpolation/InterpolatingSynthesizer.java +++ /dev/null @@ -1,203 +0,0 @@ -/** - * Copyright 2007 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.unitselection.interpolation; - -import java.io.IOException; -import java.io.PrintWriter; -import java.io.StringWriter; -import java.util.Iterator; -import java.util.List; - -import javax.sound.sampled.AudioInputStream; - -import marytts.datatypes.MaryXML; -import marytts.exceptions.SynthesisException; -import marytts.modules.synthesis.Voice; -import marytts.modules.synthesis.WaveformSynthesizer; -import marytts.signalproc.process.FramewiseMerger; -import marytts.signalproc.process.LSFInterpolator; -import marytts.unitselection.UnitSelectionVoice; -import marytts.unitselection.concat.BaseUnitConcatenator; -import marytts.unitselection.concat.UnitConcatenator; -import marytts.unitselection.select.SelectedUnit; -import marytts.unitselection.select.UnitSelector; -import marytts.util.MaryUtils; -import marytts.util.data.BufferedDoubleDataSource; -import marytts.util.data.Datagram; -import marytts.util.data.DoubleDataSource; -import marytts.util.data.audio.AudioDoubleDataSource; -import marytts.util.data.audio.DDSAudioInputStream; -import marytts.util.dom.MaryDomUtils; - -import org.apache.log4j.Logger; -import org.w3c.dom.Element; - -/** - * @author marc - * - */ -public class InterpolatingSynthesizer implements WaveformSynthesizer { - protected Logger logger; - - /** - * - */ - public InterpolatingSynthesizer() { - } - - /** - * Start up the waveform synthesizer. This must be called once before calling synthesize(). - */ - public void startup() throws Exception { - logger = MaryUtils.getLogger("InterpolatingSynthesizer"); - // Register interpolating voice: - Voice.registerVoice(new InterpolatingVoice(this, "interpolatingvoice")); - logger.info("started."); - } - - /** - * Perform a power-on self test by processing some example input data. - * - * @throws Error - * if the module does not work properly. - */ - public void powerOnSelfTest() throws Error { - } - - /** - * {@inheritDoc} - */ - public AudioInputStream synthesize(List tokensAndBoundaries, Voice voice, String outputParams) - throws SynthesisException { - if (tokensAndBoundaries.size() == 0) - return null; - - // 1. determine the two voices involved; - Element first = (Element) tokensAndBoundaries.get(0); - Element voiceElement = (Element) MaryDomUtils.getAncestor(first, MaryXML.VOICE); - String name = voiceElement.getAttribute("name"); - // name has the form: voice1 with XY% voice2 - // We trust that InerpolatingVoice.hasName() has done its job to verify that - // name actually has that form. - String[] parts = name.split("\\s+"); - assert parts.length == 4; - assert parts[1].equals("with"); - assert parts[2].endsWith("%"); - int percent = Integer.parseInt(parts[2].substring(0, parts[2].length() - 1)); - Voice voice1 = Voice.getVoice(parts[0]); - assert voice1 != null; - Voice voice2 = Voice.getVoice(parts[3]); - assert voice2 != null; - - // 2. do unit selection with each; - if (!(voice1 instanceof UnitSelectionVoice)) { - throw new IllegalArgumentException("Voices of type " + voice.getClass().getName() + " not supported!"); - } - if (!(voice2 instanceof UnitSelectionVoice)) { - throw new IllegalArgumentException("Voices of type " + voice.getClass().getName() + " not supported!"); - } - UnitSelectionVoice usv1 = (UnitSelectionVoice) voice1; - UnitSelectionVoice usv2 = (UnitSelectionVoice) voice2; - - UnitSelector unitSel1 = usv1.getUnitSelector(); - List selectedUnits1 = unitSel1.selectUnits(tokensAndBoundaries, voice); - UnitSelector unitSel2 = usv2.getUnitSelector(); - List selectedUnits2 = unitSel2.selectUnits(tokensAndBoundaries, voice); - assert selectedUnits1.size() == selectedUnits2.size() : "Unexpected difference in number of units: " - + selectedUnits1.size() + " vs. " + selectedUnits2.size(); - int numUnits = selectedUnits1.size(); - - // 3. do unit concatenation with each, retrieve actual unit durations from list of units; - UnitConcatenator unitConcatenator1 = usv1.getConcatenator(); - AudioInputStream audio1; - try { - audio1 = unitConcatenator1.getAudio(selectedUnits1); - } catch (IOException ioe) { - StringWriter sw = new StringWriter(); - PrintWriter pw = new PrintWriter(sw); - for (Iterator selIt = selectedUnits1.iterator(); selIt.hasNext();) - pw.println(selIt.next()); - throw new SynthesisException("For voice " + voice1.getName() + ", problems generating audio for unit chain: " - + sw.toString(), ioe); - } - DoubleDataSource audioSource1 = new AudioDoubleDataSource(audio1); - UnitConcatenator unitConcatenator2 = usv2.getConcatenator(); - AudioInputStream audio2; - try { - audio2 = unitConcatenator2.getAudio(selectedUnits2); - } catch (IOException ioe) { - StringWriter sw = new StringWriter(); - PrintWriter pw = new PrintWriter(sw); - for (Iterator selIt = selectedUnits2.iterator(); selIt.hasNext();) - pw.println(selIt.next()); - throw new SynthesisException("For voice " + voice2.getName() + ", problems generating audio for unit chain: " - + sw.toString(), ioe); - } - DoubleDataSource audioSource2 = new AudioDoubleDataSource(audio2); - // Retrieve actual durations from list of units: - int sampleRate1 = (int) usv1.dbAudioFormat().getSampleRate(); - double[] label1 = new double[numUnits]; - double t1 = 0; - int sampleRate2 = (int) usv2.dbAudioFormat().getSampleRate(); - double[] label2 = new double[numUnits]; - double t2 = 0; - for (int i = 0; i < numUnits; i++) { - SelectedUnit u1 = selectedUnits1.get(i); - SelectedUnit u2 = selectedUnits2.get(i); - BaseUnitConcatenator.UnitData ud1 = (BaseUnitConcatenator.UnitData) u1.getConcatenationData(); - int unitDuration1 = ud1.getUnitDuration(); - if (unitDuration1 < 0) { // was not set by the unit concatenator, have to count ourselves - unitDuration1 = 0; - Datagram[] d = ud1.getFrames(); - for (int id = 0; id < d.length; id++) { - unitDuration1 += d[id].getDuration(); - } - } - t1 += unitDuration1 / (float) sampleRate1; - label1[i] = t1; - BaseUnitConcatenator.UnitData ud2 = (BaseUnitConcatenator.UnitData) u2.getConcatenationData(); - int unitDuration2 = ud2.getUnitDuration(); - if (unitDuration2 < 0) { // was not set by the unit concatenator, have to count ourselves - unitDuration2 = 0; - Datagram[] d = ud2.getFrames(); - for (int id = 0; id < d.length; id++) { - unitDuration2 += d[id].getDuration(); - } - } - t2 += unitDuration2 / (float) sampleRate2; - label2[i] = t2; - logger.debug(usv1.getName() + " [" + u1.getTarget() + "] " + label1[i] + " -- " + usv2.getName() + " [" - + u2.getTarget() + "] " + label2[i]); - } - - // 4. with these unit durations, run the LSFInterpolator on the two audio streams. - int frameLength = Integer.getInteger("signalproc.lpcanalysisresynthesis.framelength", 512).intValue(); - int predictionOrder = Integer.getInteger("signalproc.lpcanalysisresynthesis.predictionorder", 20).intValue(); - double r = (double) percent / 100; - assert r >= 0; - assert r <= 1; - FramewiseMerger foas = new FramewiseMerger(audioSource1, frameLength, sampleRate1, new BufferedDoubleDataSource(label1), - audioSource2, sampleRate2, new BufferedDoubleDataSource(label2), new LSFInterpolator(predictionOrder, r)); - DDSAudioInputStream outputAudio = new DDSAudioInputStream(new BufferedDoubleDataSource(foas), audio1.getFormat()); - - return outputAudio; - } - -} diff --git a/marytts-runtime/src/main/java/marytts/unitselection/interpolation/InterpolatingVoice.java b/marytts-runtime/src/main/java/marytts/unitselection/interpolation/InterpolatingVoice.java deleted file mode 100644 index 903b5c4a29..0000000000 --- a/marytts-runtime/src/main/java/marytts/unitselection/interpolation/InterpolatingVoice.java +++ /dev/null @@ -1,119 +0,0 @@ -/** - * Copyright 2007 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.unitselection.interpolation; - -import java.util.Locale; - -import javax.sound.sampled.AudioFormat; - -import marytts.exceptions.MaryConfigurationException; -import marytts.modules.phonemiser.Allophone; -import marytts.modules.phonemiser.AllophoneSet; -import marytts.modules.synthesis.Voice; - -/** - * @author marc - * - */ -public class InterpolatingVoice extends Voice { - - public static boolean isInterpolatingVoiceName(String name) { - if (name == null) - return false; - String[] parts = name.split("\\s+"); - if (parts.length != 4) - return false; - if (!parts[1].equals("with")) - return false; - if (!parts[2].endsWith("%")) - return false; - int percent; - try { - percent = Integer.parseInt(parts[2].substring(0, parts[2].length() - 1)); - } catch (NumberFormatException nfe) { - return false; - } - if (Voice.getVoice(parts[0]) == null) - return false; - if (Voice.getVoice(parts[3]) == null) - return false; - return true; - } - - protected Voice firstVoice = null; - - public InterpolatingVoice(InterpolatingSynthesizer is, String name) throws MaryConfigurationException { - super(name, null, null, is, null); - if (isInterpolatingVoiceName(name)) { - String[] parts = name.split("\\s+"); - firstVoice = Voice.getVoice(parts[0]); - } - } - - /** - * Determine whether this voice has the given name. For the InterpolatingVoice, the meaning of the name is different from a - * "normal" voice. It is a specification of how to interpolate two voices. The syntax is:
- * voice1 with XY% voice2
- *
- * where voice1 and voice2 must be existing voices, and XY is an integer between 0 and 100. - * - * @return true if name matches the specification, false otherwise - */ - /* - * public boolean hasName(String name) { if (name == null) return false; String[] parts = name.split("\\s+"); if (parts.length - * != 4) return false; if (!parts[1].equals("with")) return false; if (!parts[2].endsWith("%")) return false; int percent; try - * { percent = Integer.parseInt(parts[2].substring(0, parts[2].length()-1)); } catch (NumberFormatException nfe) { return - * false; } if (Voice.getVoice(parts[0]) == null) return false; if (Voice.getVoice(parts[3]) == null) return false; return - * true; } - */ - - // Forward most of the public methods which are meaningful in a unit selection context to firstVoice: - - public AllophoneSet getAllophoneSet() { - if (firstVoice == null) - return null; - return firstVoice.getAllophoneSet(); - } - - public Allophone getAllophone(String phoneSymbol) { - if (firstVoice == null) - return null; - return firstVoice.getAllophone(phoneSymbol); - } - - public Locale getLocale() { - if (firstVoice == null) - return null; - return firstVoice.getLocale(); - } - - public AudioFormat dbAudioFormat() { - if (firstVoice == null) - return null; - return firstVoice.dbAudioFormat(); - } - - public Gender gender() { - if (firstVoice == null) - return null; - return firstVoice.gender(); - } - -} diff --git a/marytts-runtime/src/main/java/marytts/unitselection/select/DiphoneFFRTargetCostFunction.java b/marytts-runtime/src/main/java/marytts/unitselection/select/DiphoneFFRTargetCostFunction.java deleted file mode 100644 index 9f14108836..0000000000 --- a/marytts-runtime/src/main/java/marytts/unitselection/select/DiphoneFFRTargetCostFunction.java +++ /dev/null @@ -1,142 +0,0 @@ -/** - * Copyright 2006 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.unitselection.select; - -import java.io.IOException; -import java.io.InputStream; - -import marytts.exceptions.MaryConfigurationException; -import marytts.features.FeatureDefinition; -import marytts.features.FeatureProcessorManager; -import marytts.features.FeatureVector; -import marytts.unitselection.data.DiphoneUnit; -import marytts.unitselection.data.FeatureFileReader; -import marytts.unitselection.data.HalfPhoneFeatureFileReader; -import marytts.unitselection.data.Unit; - -public class DiphoneFFRTargetCostFunction implements TargetCostFunction { - protected FFRTargetCostFunction tcfForHalfphones; - - public DiphoneFFRTargetCostFunction() { - } - - /** - * Initialise the data needed to do a target cost computation. - * - * @param featureFileName - * name of a file containing the unit features - * @param weightsFile - * an optional weights file -- if non-null, contains feature weights that override the ones present in the feature - * file. - * @param featProc - * a feature processor manager which can provide feature processors to compute the features for a target at run - * time - * @throws IOException - */ - @Override - public void load(String featureFileName, InputStream weightsStream, FeatureProcessorManager featProc) throws IOException, - MaryConfigurationException { - FeatureFileReader ffr = FeatureFileReader.getFeatureFileReader(featureFileName); - load(ffr, weightsStream, featProc); - } - - @Override - public void load(FeatureFileReader ffr, InputStream weightsStream, FeatureProcessorManager featProc) throws IOException { - if (ffr instanceof HalfPhoneFeatureFileReader) { - tcfForHalfphones = new HalfPhoneFFRTargetCostFunction(); - } else { - tcfForHalfphones = new FFRTargetCostFunction(); - } - tcfForHalfphones.load(ffr, weightsStream, featProc); - } - - /** - * Provide access to the Feature Definition used. - * - * @return the feature definition object. - */ - public FeatureDefinition getFeatureDefinition() { - return tcfForHalfphones.getFeatureDefinition(); - } - - /** - * Get the string representation of the feature value associated with the given unit - * - * @param unit - * the unit whose feature value is requested - * @param featureName - * name of the feature requested - * @return a string representation of the feature value - * @throws IllegalArgumentException - * if featureName is not a known feature - */ - public String getFeature(Unit unit, String featureName) { - return tcfForHalfphones.getFeature(unit, featureName); - } - - public FeatureVector getFeatureVector(Unit unit) { - return tcfForHalfphones.featureVectors[unit.index]; - } - - /** - * Compute the goodness-of-fit of a given unit for a given target. - * - * @param target - * @param unit - * @return a non-negative number; smaller values mean better fit, i.e. smaller cost. - */ - public double cost(Target target, Unit unit) { - if (target instanceof HalfPhoneTarget) - return tcfForHalfphones.cost(target, unit); - if (!(target instanceof DiphoneTarget)) - throw new IllegalArgumentException("This target cost function can only be called for diphone and half-phone targets!"); - if (!(unit instanceof DiphoneUnit)) - throw new IllegalArgumentException("Diphone targets need diphone units!"); - DiphoneTarget dt = (DiphoneTarget) target; - DiphoneUnit du = (DiphoneUnit) unit; - return tcfForHalfphones.cost(dt.left, du.left) + tcfForHalfphones.cost(dt.right, du.right); - } - - /** - * Compute the features for a given target, and store them in the target. - * - * @param target - * the target for which to compute the features - * @see Target#getFeatureVector() - */ - public void computeTargetFeatures(Target target) { - if (!(target instanceof DiphoneTarget)) { - tcfForHalfphones.computeTargetFeatures(target); - } else { - DiphoneTarget dt = (DiphoneTarget) target; - tcfForHalfphones.computeTargetFeatures(dt.left); - tcfForHalfphones.computeTargetFeatures(dt.right); - - } - } - - public FeatureVector[] getFeatureVectors() { - if (tcfForHalfphones != null) { - return tcfForHalfphones.getFeatureVectors(); - } - return null; - } - -} diff --git a/marytts-runtime/src/main/java/marytts/unitselection/select/DiphoneTarget.java b/marytts-runtime/src/main/java/marytts/unitselection/select/DiphoneTarget.java deleted file mode 100644 index 39fc4f3259..0000000000 --- a/marytts-runtime/src/main/java/marytts/unitselection/select/DiphoneTarget.java +++ /dev/null @@ -1,71 +0,0 @@ -/** - * Copyright 2006 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.unitselection.select; - -import marytts.features.FeatureVector; -import marytts.modules.phonemiser.Allophone; - -import org.w3c.dom.Element; - -public class DiphoneTarget extends Target { - public final HalfPhoneTarget left; - public final HalfPhoneTarget right; - - public DiphoneTarget(HalfPhoneTarget left, HalfPhoneTarget right) { - super(null, null); - this.name = left.name.substring(0, left.name.lastIndexOf("_")) + "-" - + right.name.substring(0, right.name.lastIndexOf("_")); - assert left.isRightHalf(); // the left half of this diphone must be the right half of a phone - assert right.isLeftHalf(); - this.left = left; - this.right = right; - } - - @Override - public Element getMaryxmlElement() { - throw new IllegalStateException("This method should not be called for DiphoneTargets."); - } - - public FeatureVector getFeatureVector() { - throw new IllegalStateException("This method should not be called for DiphoneTargets."); - } - - public void setFeatureVector(FeatureVector featureVector) { - throw new IllegalStateException("This method should not be called for DiphoneTargets."); - } - - public float getTargetDurationInSeconds() { - throw new IllegalStateException("This method should not be called for DiphoneTargets."); - } - - /** - * Determine whether this target is a silence target - * - * @return true if the target represents silence, false otherwise - */ - public boolean isSilence() { - throw new IllegalStateException("This method should not be called for DiphoneTargets."); - } - - public Allophone getAllophone() { - throw new IllegalStateException("This method should not be called for DiphoneTargets."); - } - -} diff --git a/marytts-runtime/src/main/java/marytts/unitselection/select/DiphoneUnitSelector.java b/marytts-runtime/src/main/java/marytts/unitselection/select/DiphoneUnitSelector.java deleted file mode 100644 index 756c6d1ff6..0000000000 --- a/marytts-runtime/src/main/java/marytts/unitselection/select/DiphoneUnitSelector.java +++ /dev/null @@ -1,69 +0,0 @@ -/** - * Copyright 2006 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.unitselection.select; - -import java.util.ArrayList; -import java.util.List; - -import marytts.unitselection.data.UnitDatabase; - -import org.w3c.dom.Element; - -public class DiphoneUnitSelector extends UnitSelector { - - /** - * Initialise the unit selector. Need to call load() separately. - * - * @see #load(UnitDatabase) - */ - public DiphoneUnitSelector() throws Exception { - super(); - } - - /** - * Create the list of targets from the XML elements to synthesize. - * - * @param segmentsAndBoundaries - * a list of MaryXML phone and boundary elements - * @return a list of Target objects - */ - protected List createTargets(List segmentsAndBoundaries) { - List targets = new ArrayList(); - // TODO: how can we know the silence symbol here? - String silenceSymbol = "_"; // in sampa - - // Insert an initial silence with duration 0 (needed as context) - HalfPhoneTarget prev = new HalfPhoneTarget(silenceSymbol + "_R", null, false); - for (Element sOrB : segmentsAndBoundaries) { - String phone = getPhoneSymbol(sOrB); - HalfPhoneTarget leftHalfPhone = new HalfPhoneTarget(phone + "_L", sOrB, true); // left half - HalfPhoneTarget rightHalfPhone = new HalfPhoneTarget(phone + "_R", sOrB, false); // right half - targets.add(new DiphoneTarget(prev, leftHalfPhone)); - prev = rightHalfPhone; - } - // Make sure there is a final silence - if (!prev.isSilence()) { - HalfPhoneTarget silence = new HalfPhoneTarget(silenceSymbol + "_L", null, true); - targets.add(new DiphoneTarget(prev, silence)); - } - return targets; - } - -} diff --git a/marytts-runtime/src/main/java/marytts/unitselection/select/FFRTargetCostFunction.java b/marytts-runtime/src/main/java/marytts/unitselection/select/FFRTargetCostFunction.java deleted file mode 100644 index 2c1a56800f..0000000000 --- a/marytts-runtime/src/main/java/marytts/unitselection/select/FFRTargetCostFunction.java +++ /dev/null @@ -1,386 +0,0 @@ -/** - * Copyright 2006 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.unitselection.select; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; - -import marytts.exceptions.MaryConfigurationException; -import marytts.features.FeatureDefinition; -import marytts.features.FeatureProcessorManager; -import marytts.features.FeatureVector; -import marytts.features.TargetFeatureComputer; -import marytts.server.MaryProperties; -import marytts.signalproc.display.Histogram; -import marytts.unitselection.data.FeatureFileReader; -import marytts.unitselection.data.Unit; -import marytts.unitselection.weightingfunctions.WeightFunc; -import marytts.unitselection.weightingfunctions.WeightFunctionManager; -import marytts.util.MaryUtils; - -public class FFRTargetCostFunction implements TargetCostFunction { - protected WeightFunc[] weightFunction; - protected TargetFeatureComputer targetFeatureComputer; - protected FeatureVector[] featureVectors; - protected FeatureDefinition featureDefinition; - protected boolean[] weightsNonZero; - - protected boolean debugShowCostGraph = false; - protected double[] cumulWeightedCosts = null; - protected int nCostComputations = 0; - - public FFRTargetCostFunction() { - } - - /** - * Compute the goodness-of-fit of a given unit for a given target. - * - * @param target - * @param unit - * @return a non-negative number; smaller values mean better fit, i.e. smaller cost. - */ - public double cost(Target target, Unit unit) { - return cost(target, unit, featureDefinition, weightFunction); - } - - protected double cost(Target target, Unit unit, FeatureDefinition weights, WeightFunc[] weightFunctions) { - nCostComputations++; // for debug - FeatureVector targetFeatures = target.getFeatureVector(); - assert targetFeatures != null : "Target " + target + " does not have pre-computed feature vector"; - FeatureVector unitFeatures = featureVectors[unit.index]; - int nBytes = targetFeatures.byteValuedDiscreteFeatures.length; - int nShorts = targetFeatures.shortValuedDiscreteFeatures.length; - int nFloats = targetFeatures.continuousFeatures.length; - assert nBytes == unitFeatures.byteValuedDiscreteFeatures.length; - assert nShorts == unitFeatures.shortValuedDiscreteFeatures.length; - assert nFloats == unitFeatures.continuousFeatures.length; - - float[] weightVector = weights.getFeatureWeights(); - // Now the actual computation - double cost = 0; - // byte-valued features: - if (nBytes > 0) { - for (int i = 0; i < nBytes; i++) { - if (weightsNonZero[i]) { - float weight = weightVector[i]; - if (featureDefinition.hasSimilarityMatrix(i)) { - byte targetFeatValueIndex = targetFeatures.byteValuedDiscreteFeatures[i]; - byte unitFeatValueIndex = unitFeatures.byteValuedDiscreteFeatures[i]; - float similarity = featureDefinition.getSimilarity(i, unitFeatValueIndex, targetFeatValueIndex); - cost += similarity * weight; - if (debugShowCostGraph) - cumulWeightedCosts[i] += similarity * weight; - } else if (targetFeatures.byteValuedDiscreteFeatures[i] != unitFeatures.byteValuedDiscreteFeatures[i]) { - cost += weight; - if (debugShowCostGraph) - cumulWeightedCosts[i] += weight; - } - } - } - } - // short-valued features: - if (nShorts > 0) { - for (int i = nBytes, n = nBytes + nShorts; i < n; i++) { - if (weightsNonZero[i]) { - float weight = weightVector[i]; - // if (targetFeatures.getShortFeature(i) != unitFeatures.getShortFeature(i)) { - if (targetFeatures.shortValuedDiscreteFeatures[i - nBytes] != unitFeatures.shortValuedDiscreteFeatures[i - - nBytes]) { - cost += weight; - if (debugShowCostGraph) - cumulWeightedCosts[i] += weight; - } - } - } - } - // continuous features: - if (nFloats > 0) { - int nDiscrete = nBytes + nShorts; - for (int i = nDiscrete, n = nDiscrete + nFloats; i < n; i++) { - if (weightsNonZero[i]) { - float weight = weightVector[i]; - // float a = targetFeatures.getContinuousFeature(i); - float a = targetFeatures.continuousFeatures[i - nDiscrete]; - // float b = unitFeatures.getContinuousFeature(i); - float b = unitFeatures.continuousFeatures[i - nDiscrete]; - // if (!Float.isNaN(a) && !Float.isNaN(b)) { - // Implementation of isNaN() is: (v != v). - if (!(a != a) && !(b != b)) { - double myCost = weightFunctions[i - nDiscrete].cost(a, b); - cost += weight * myCost; - if (debugShowCostGraph) { - cumulWeightedCosts[i] += weight * myCost; - } - } // and if it is NaN, simply compute no cost - } - } - } - return cost; - } - - /** - * Compute the goodness-of-fit between given unit and given target for a given feature - * - * @param target - * target unit - * @param unit - * candidate unit - * @param featureName - * feature name - * @return a non-negative number; smaller values mean better fit, i.e. smaller cost. - * @throws IllegalArgumentException - * if featureName not available in featureDefinition - */ - public double featureCost(Target target, Unit unit, String featureName) { - return featureCost(target, unit, featureName, featureDefinition, weightFunction); - } - - protected double featureCost(Target target, Unit unit, String featureName, FeatureDefinition weights, - WeightFunc[] weightFunctions) { - if (!this.featureDefinition.hasFeature(featureName)) { - throw new IllegalArgumentException("this feature does not exists in feature definition"); - } - - FeatureVector targetFeatures = target.getFeatureVector(); - assert targetFeatures != null : "Target " + target + " does not have pre-computed feature vector"; - FeatureVector unitFeatures = featureVectors[unit.index]; - int nBytes = targetFeatures.byteValuedDiscreteFeatures.length; - int nShorts = targetFeatures.shortValuedDiscreteFeatures.length; - int nFloats = targetFeatures.continuousFeatures.length; - assert nBytes == unitFeatures.byteValuedDiscreteFeatures.length; - assert nShorts == unitFeatures.shortValuedDiscreteFeatures.length; - assert nFloats == unitFeatures.continuousFeatures.length; - - int featureIndex = this.featureDefinition.getFeatureIndex(featureName); - float[] weightVector = weights.getFeatureWeights(); - double cost = 0; - - if (featureIndex < nBytes) { - if (weightsNonZero[featureIndex]) { - float weight = weightVector[featureIndex]; - if (featureDefinition.hasSimilarityMatrix(featureIndex)) { - byte targetFeatValueIndex = targetFeatures.byteValuedDiscreteFeatures[featureIndex]; - byte unitFeatValueIndex = unitFeatures.byteValuedDiscreteFeatures[featureIndex]; - float similarity = featureDefinition.getSimilarity(featureIndex, unitFeatValueIndex, targetFeatValueIndex); - cost = similarity * weight; - if (debugShowCostGraph) - cumulWeightedCosts[featureIndex] += similarity * weight; - } else if (targetFeatures.byteValuedDiscreteFeatures[featureIndex] != unitFeatures.byteValuedDiscreteFeatures[featureIndex]) { - cost = weight; - if (debugShowCostGraph) - cumulWeightedCosts[featureIndex] += weight; - } - } - } else if (featureIndex < nShorts + nBytes) { - if (weightsNonZero[featureIndex]) { - float weight = weightVector[featureIndex]; - // if (targetFeatures.getShortFeature(i) != unitFeatures.getShortFeature(i)) { - if (targetFeatures.shortValuedDiscreteFeatures[featureIndex - nBytes] != unitFeatures.shortValuedDiscreteFeatures[featureIndex - - nBytes]) { - cost = weight; - if (debugShowCostGraph) - cumulWeightedCosts[featureIndex] += weight; - } - } - } else { - int nDiscrete = nBytes + nShorts; - if (weightsNonZero[featureIndex]) { - float weight = weightVector[featureIndex]; - // float a = targetFeatures.getContinuousFeature(i); - float a = targetFeatures.continuousFeatures[featureIndex - nDiscrete]; - // float b = unitFeatures.getContinuousFeature(i); - float b = unitFeatures.continuousFeatures[featureIndex - nDiscrete]; - // if (!Float.isNaN(a) && !Float.isNaN(b)) { - // Implementation of isNaN() is: (v != v). - if (!(a != a) && !(b != b)) { - double myCost = weightFunctions[featureIndex - nDiscrete].cost(a, b); - cost = weight * myCost; - if (debugShowCostGraph) { - cumulWeightedCosts[featureIndex] += weight * myCost; - } - } // and if it is NaN, simply compute no cost - } - } - return cost; - } - - /** - * Initialise the data needed to do a target cost computation. - * - * @param featureFileName - * name of a file containing the unit features - * @param weightsFile - * an optional weights file -- if non-null, contains feature weights that override the ones present in the feature - * file. - * @param featProc - * a feature processor manager which can provide feature processors to compute the features for a target at run - * time - * @throws IOException - * @throws MaryConfigurationException - */ - @Override - public void load(String featureFileName, InputStream weightsStream, FeatureProcessorManager featProc) throws IOException, - MaryConfigurationException { - FeatureFileReader ffr = FeatureFileReader.getFeatureFileReader(featureFileName); - load(ffr, weightsStream, featProc); - } - - @Override - public void load(FeatureFileReader ffr, InputStream weightsStream, FeatureProcessorManager featProc) throws IOException { - this.featureDefinition = ffr.getFeatureDefinition(); - this.featureVectors = ffr.getFeatureVectors(); - if (weightsStream != null) { - MaryUtils.getLogger("TargetCostFeatures").debug("Overwriting target cost weights from file"); - // overwrite weights from file - - FeatureDefinition newWeights = new FeatureDefinition( - new BufferedReader(new InputStreamReader(weightsStream, "UTF-8")), true); - if (!newWeights.featureEquals(featureDefinition)) { - throw new IOException("Weights file: feature definition incompatible with feature file"); - } - featureDefinition = newWeights; - } - weightFunction = new WeightFunc[featureDefinition.getNumberOfContinuousFeatures()]; - WeightFunctionManager wfm = new WeightFunctionManager(); - int nDiscreteFeatures = featureDefinition.getNumberOfByteFeatures() + featureDefinition.getNumberOfShortFeatures(); - for (int i = 0; i < weightFunction.length; i++) { - String weightFunctionName = featureDefinition.getWeightFunctionName(nDiscreteFeatures + i); - if ("".equals(weightFunctionName)) - weightFunction[i] = wfm.getWeightFunction("linear"); - else - weightFunction[i] = wfm.getWeightFunction(weightFunctionName); - } - // TODO: If the target feature computer had direct access to the feature definition, it could do some consistency checking - this.targetFeatureComputer = new TargetFeatureComputer(featProc, featureDefinition.getFeatureNames()); - - rememberWhichWeightsAreNonZero(); - - if (MaryProperties.getBoolean("debug.show.cost.graph")) { - debugShowCostGraph = true; - cumulWeightedCosts = new double[featureDefinition.getNumberOfFeatures()]; - TargetCostReporter tcr2 = new TargetCostReporter(cumulWeightedCosts); - tcr2.showInJFrame("Average weighted target costs", false, false); - tcr2.start(); - } - } - - protected void rememberWhichWeightsAreNonZero() { - // remember which weights are non-zero - weightsNonZero = new boolean[featureDefinition.getNumberOfFeatures()]; - for (int i = 0, n = featureDefinition.getNumberOfFeatures(); i < n; i++) { - weightsNonZero[i] = (featureDefinition.getWeight(i) > 0); - } - } - - /** - * Compute the features for a given target, and store them in the target. - * - * @param target - * the target for which to compute the features - * @see Target#getFeatureVector() - */ - public void computeTargetFeatures(Target target) { - FeatureVector fv = targetFeatureComputer.computeFeatureVector(target); - target.setFeatureVector(fv); - } - - /** - * Look up the features for a given unit. - * - * @param unit - * a unit in the database - * @return the FeatureVector for target cost computation associated to this unit - */ - public FeatureVector getFeatureVector(Unit unit) { - return featureVectors[unit.index]; - } - - /** - * Get the string representation of the feature value associated with the given unit - * - * @param unit - * the unit whose feature value is requested - * @param featureName - * name of the feature requested - * @return a string representation of the feature value - * @throws IllegalArgumentException - * if featureName is not a known feature - */ - public String getFeature(Unit unit, String featureName) { - int featureIndex = featureDefinition.getFeatureIndex(featureName); - if (featureDefinition.isByteFeature(featureIndex)) { - byte value = featureVectors[unit.index].getByteFeature(featureIndex); - return featureDefinition.getFeatureValueAsString(featureIndex, value); - } else if (featureDefinition.isShortFeature(featureIndex)) { - short value = featureVectors[unit.index].getShortFeature(featureIndex); - return featureDefinition.getFeatureValueAsString(featureIndex, value); - } else { // continuous -- return float as string - float value = featureVectors[unit.index].getContinuousFeature(featureIndex); - return String.valueOf(value); - } - } - - public FeatureDefinition getFeatureDefinition() { - return featureDefinition; - } - - public class TargetCostReporter extends Histogram { - private double[] data; - private int lastN = 0; - - public TargetCostReporter(double[] data) { - super(0, 1, data); - this.data = data; - } - - public void start() { - new Thread() { - public void run() { - while (isVisible()) { - try { - Thread.sleep(500); - } catch (InterruptedException ie) { - } - updateGraph(); - } - } - }.start(); - } - - protected void updateGraph() { - if (nCostComputations == lastN) - return; - lastN = nCostComputations; - double[] newCosts = new double[data.length]; - for (int i = 0; i < newCosts.length; i++) { - newCosts[i] = data[i] / nCostComputations; - } - updateData(0, 1, newCosts); - repaint(); - } - } - - public FeatureVector[] getFeatureVectors() { - return featureVectors; - } - -} diff --git a/marytts-runtime/src/main/java/marytts/unitselection/select/HalfPhoneFFRTargetCostFunction.java b/marytts-runtime/src/main/java/marytts/unitselection/select/HalfPhoneFFRTargetCostFunction.java deleted file mode 100644 index cee861b59b..0000000000 --- a/marytts-runtime/src/main/java/marytts/unitselection/select/HalfPhoneFFRTargetCostFunction.java +++ /dev/null @@ -1,243 +0,0 @@ -/** - * Copyright 2006 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.unitselection.select; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStreamReader; - -import marytts.exceptions.MaryConfigurationException; -import marytts.features.FeatureDefinition; -import marytts.features.FeatureProcessorManager; -import marytts.features.FeatureVector; -import marytts.features.TargetFeatureComputer; -import marytts.server.MaryProperties; -import marytts.signalproc.display.Histogram; -import marytts.unitselection.data.FeatureFileReader; -import marytts.unitselection.data.HalfPhoneFeatureFileReader; -import marytts.unitselection.data.Unit; -import marytts.unitselection.weightingfunctions.WeightFunc; -import marytts.unitselection.weightingfunctions.WeightFunctionManager; -import marytts.util.MaryUtils; - -public class HalfPhoneFFRTargetCostFunction extends FFRTargetCostFunction { - protected FeatureDefinition leftWeights; - protected FeatureDefinition rightWeights; - protected WeightFunc[] leftWeightFunction; - protected WeightFunc[] rightWeightFunction; - - public HalfPhoneFFRTargetCostFunction() { - } - - /** - * Compute the goodness-of-fit of a given unit for a given target. - * - * @param target - * @param unit - * @return a non-negative number; smaller values mean better fit, i.e. smaller cost. - */ - public double cost(Target target, Unit unit) { - if (!(target instanceof HalfPhoneTarget)) - throw new IllegalArgumentException("This target cost function can only be called for half-phone targets!"); - HalfPhoneTarget hpTarget = (HalfPhoneTarget) target; - boolean isLeftHalf = hpTarget.isLeftHalf(); - FeatureDefinition weights = isLeftHalf ? leftWeights : rightWeights; - WeightFunc[] weightFunctions = isLeftHalf ? leftWeightFunction : rightWeightFunction; - return cost(target, unit, weights, weightFunctions); - } - - /** - * Initialise the data needed to do a target cost computation. - * - * @param featureFileName - * name of a file containing the unit features - * @param weightsFile - * an optional string containing weights file names -- if non-null, contains two file names separated by the - * character '|', pointing to feature weights files for left and right units, respectively, that override the ones - * present in the feature file. - * @param featProc - * a feature processor manager which can provide feature processors to compute the features for a target at run - * time - * @throws IOException - */ - public void load(String featureFileName, String weightsFile, FeatureProcessorManager featProc) throws IOException, - MaryConfigurationException { - HalfPhoneFeatureFileReader ffr = new HalfPhoneFeatureFileReader(featureFileName); - load(ffr, weightsFile, featProc); - } - - public void load(FeatureFileReader featureFileReader, String weightsFile, FeatureProcessorManager featProc) - throws IOException { - if (!(featureFileReader instanceof HalfPhoneFeatureFileReader)) - throw new IllegalArgumentException("Featurefilereader must be a HalfPhoneFeatureFileReader"); - HalfPhoneFeatureFileReader ffr = (HalfPhoneFeatureFileReader) featureFileReader; - this.leftWeights = ffr.getLeftWeights(); - this.featureDefinition = this.leftWeights; - this.rightWeights = ffr.getRightWeights(); - this.featureVectors = ffr.getFeatureVectors(); - - if (weightsFile != null) { - MaryUtils.getLogger("TargetCostFeatures").debug("Overwriting target cost weights from file " + weightsFile); - String[] weightsFiles = weightsFile.split("\\|"); - if (weightsFiles.length != 2) - throw new IllegalArgumentException( - "Parameter weightsFile should contain exactly two fields separated by a '|' character -- instead, it is: '" - + weightsFile + "'"); - File leftF = new File(weightsFiles[0].trim()); - File rightF; - // If the second weights file has no path, it is in the same directory as the first - if (weightsFiles[1].indexOf("/") == -1 && weightsFiles[1].indexOf("\\") == -1) { - File dir = leftF.getParentFile(); - rightF = new File(dir, weightsFiles[1].trim()); - } else { - rightF = new File(weightsFiles[1].trim()); - } - - // overwrite weights from files - FeatureDefinition newLeftWeights = new FeatureDefinition(new BufferedReader(new InputStreamReader( - new FileInputStream(leftF), "UTF-8")), true); - if (!newLeftWeights.featureEquals(leftWeights)) { - throw new IOException("Weights file '" + leftF + "': feature definition incompatible with feature file"); - } - leftWeights = newLeftWeights; - FeatureDefinition newRightWeights = new FeatureDefinition(new BufferedReader(new InputStreamReader( - new FileInputStream(rightF), "UTF-8")), true); - if (!newRightWeights.featureEquals(rightWeights)) { - throw new IOException("Weights file '" + rightF + "': feature definition incompatible with feature file"); - } - rightWeights = newRightWeights; - } - WeightFunctionManager wfm = new WeightFunctionManager(); - WeightFunc linear = wfm.getWeightFunction("linear"); - int nDiscreteFeatures = leftWeights.getNumberOfByteFeatures() + leftWeights.getNumberOfShortFeatures(); - leftWeightFunction = new WeightFunc[leftWeights.getNumberOfContinuousFeatures()]; - rightWeightFunction = new WeightFunc[leftWeightFunction.length]; - for (int i = 0; i < leftWeightFunction.length; i++) { - String weightFunctionName = leftWeights.getWeightFunctionName(nDiscreteFeatures + i); - if ("".equals(weightFunctionName)) - leftWeightFunction[i] = linear; - else - leftWeightFunction[i] = wfm.getWeightFunction(weightFunctionName); - weightFunctionName = rightWeights.getWeightFunctionName(nDiscreteFeatures + i); - if ("".equals(weightFunctionName)) - rightWeightFunction[i] = linear; - else - rightWeightFunction[i] = wfm.getWeightFunction(weightFunctionName); - } - // TODO: If the target feature computer had direct access to the feature definition, it could do some consistency checking - this.targetFeatureComputer = new TargetFeatureComputer(featProc, leftWeights.getFeatureNames()); - - rememberWhichWeightsAreNonZero(); - - if (MaryProperties.getBoolean("debug.show.cost.graph")) { - debugShowCostGraph = true; - cumulWeightedCosts = new double[featureDefinition.getNumberOfFeatures()]; - TargetCostReporter tcr2 = new TargetCostReporter(cumulWeightedCosts); - tcr2.showInJFrame("Average weighted target costs", false, false); - tcr2.start(); - } - } - - /** - * Compute the features for a given target, and store them in the target. - * - * @param target - * the target for which to compute the features - * @see Target#getFeatureVector() - */ - public void computeTargetFeatures(Target target) { - FeatureVector fv = targetFeatureComputer.computeFeatureVector(target); - target.setFeatureVector(fv); - } - - /** - * Look up the features for a given unit. - * - * @param unit - * a unit in the database - * @return the FeatureVector for target cost computation associated to this unit - */ - public FeatureVector getUnitFeatures(Unit unit) { - return featureVectors[unit.index]; - } - - /** - * Get the string representation of the feature value associated with the given unit - * - * @param unit - * the unit whose feature value is requested - * @param featureName - * name of the feature requested - * @return a string representation of the feature value - * @throws IllegalArgumentException - * if featureName is not a known feature - */ - public String getFeature(Unit unit, String featureName) { - int featureIndex = featureDefinition.getFeatureIndex(featureName); - if (featureDefinition.isByteFeature(featureIndex)) { - byte value = featureVectors[unit.index].getByteFeature(featureIndex); - return featureDefinition.getFeatureValueAsString(featureIndex, value); - } else if (featureDefinition.isShortFeature(featureIndex)) { - short value = featureVectors[unit.index].getShortFeature(featureIndex); - return featureDefinition.getFeatureValueAsString(featureIndex, value); - } else { // continuous -- return float as string - float value = featureVectors[unit.index].getContinuousFeature(featureIndex); - return String.valueOf(value); - } - } - - public class TargetCostReporter extends Histogram { - private double[] data; - private int lastN = 0; - - public TargetCostReporter(double[] data) { - super(0, 1, data); - this.data = data; - } - - public void start() { - new Thread() { - public void run() { - while (isVisible()) { - try { - Thread.sleep(500); - } catch (InterruptedException ie) { - } - updateGraph(); - } - } - }.start(); - } - - protected void updateGraph() { - if (nCostComputations == lastN) - return; - lastN = nCostComputations; - double[] newCosts = new double[data.length]; - for (int i = 0; i < newCosts.length; i++) { - newCosts[i] = data[i] / nCostComputations; - } - updateData(0, 1, newCosts); - repaint(); - } - } -} diff --git a/marytts-runtime/src/main/java/marytts/unitselection/select/HalfPhoneTarget.java b/marytts-runtime/src/main/java/marytts/unitselection/select/HalfPhoneTarget.java deleted file mode 100644 index f72ce74320..0000000000 --- a/marytts-runtime/src/main/java/marytts/unitselection/select/HalfPhoneTarget.java +++ /dev/null @@ -1,60 +0,0 @@ -/** - * Copyright 2006 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.unitselection.select; - -import org.w3c.dom.Element; - -public class HalfPhoneTarget extends Target { - protected boolean isLeftHalf; - - /** - * Create a target associated to the given segment item. - * - * @param name - * a name for the target, which may or may not coincide with the segment name. - * @param item - * the phone segment item in the Utterance structure, to be associated to this target - * @param isLeftHalf - * true if this target represents the left half of the phone, false if it represents the right half of the phone - */ - public HalfPhoneTarget(String name, Element maryxmlElement, boolean isLeftHalf) { - super(name, maryxmlElement); - this.isLeftHalf = isLeftHalf; - } - - /** - * Is this target the left half of a phone? - * - * @return - */ - public boolean isLeftHalf() { - return isLeftHalf; - } - - /** - * Is this target the right half of a phone? - * - * @return - */ - public boolean isRightHalf() { - return !isLeftHalf; - } - -} diff --git a/marytts-runtime/src/main/java/marytts/unitselection/select/HalfPhoneUnitSelector.java b/marytts-runtime/src/main/java/marytts/unitselection/select/HalfPhoneUnitSelector.java deleted file mode 100644 index 81ec261986..0000000000 --- a/marytts-runtime/src/main/java/marytts/unitselection/select/HalfPhoneUnitSelector.java +++ /dev/null @@ -1,57 +0,0 @@ -/** - * Copyright 2006 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.unitselection.select; - -import java.util.ArrayList; -import java.util.List; - -import marytts.unitselection.data.UnitDatabase; - -import org.w3c.dom.Element; - -public class HalfPhoneUnitSelector extends UnitSelector { - - /** - * Initialise the unit selector. Need to call load() separately. - * - * @see #load(UnitDatabase) - */ - public HalfPhoneUnitSelector() throws Exception { - super(); - } - - /** - * Create the list of targets from the XML elements to synthesize. - * - * @param segmentsAndBoundaries - * a list of MaryXML phone and boundary elements - * @return a list of Target objects - */ - protected List createTargets(List segmentsAndBoundaries) { - List targets = new ArrayList(); - for (Element sOrB : segmentsAndBoundaries) { - String phone = getPhoneSymbol(sOrB); - targets.add(new HalfPhoneTarget(phone + "_L", sOrB, true)); // left half - targets.add(new HalfPhoneTarget(phone + "_R", sOrB, false)); // right half - } - return targets; - } - -} diff --git a/marytts-runtime/src/main/java/marytts/unitselection/select/JoinCostFeatures.java b/marytts-runtime/src/main/java/marytts/unitselection/select/JoinCostFeatures.java deleted file mode 100644 index 98d7dacc0a..0000000000 --- a/marytts-runtime/src/main/java/marytts/unitselection/select/JoinCostFeatures.java +++ /dev/null @@ -1,623 +0,0 @@ -/** - * Copyright 2006 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.unitselection.select; - -import java.io.BufferedInputStream; -import java.io.BufferedReader; -import java.io.DataInput; -import java.io.DataInputStream; -import java.io.EOFException; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.FileReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.nio.ByteBuffer; -import java.nio.FloatBuffer; -import java.nio.channels.FileChannel; -import java.util.Vector; - -import marytts.exceptions.MaryConfigurationException; -import marytts.features.ByteValuedFeatureProcessor; -import marytts.features.MaryGenericFeatureProcessors; -import marytts.modules.phonemiser.Allophone; -import marytts.server.MaryProperties; -import marytts.signalproc.display.Histogram; -import marytts.unitselection.data.DiphoneUnit; -import marytts.unitselection.data.Unit; -import marytts.unitselection.weightingfunctions.WeightFunc; -import marytts.unitselection.weightingfunctions.WeightFunctionManager; -import marytts.util.MaryUtils; -import marytts.util.data.MaryHeader; -import marytts.util.io.StreamUtils; - -public class JoinCostFeatures implements JoinCostFunction { - - protected float wSignal; - protected float wPhonetic; - - protected boolean debugShowCostGraph = false; - protected double[] cumulWeightedSignalCosts = null; - protected int nCostComputations = 0; - - protected PrecompiledJoinCostReader precompiledCosts; - - protected JoinCostReporter jcr; - - /****************/ - /* DATA FIELDS */ - /****************/ - private MaryHeader hdr = null; - - private float[] featureWeight = null; - private WeightFunc[] weightFunction = null; - private boolean[] isLinear = null; // wether the i'th weight function is a linear function - - private float[][] leftJCF = null; - private float[][] rightJCF = null; - - /****************/ - /* CONSTRUCTORS */ - /****************/ - - /** - * Empty constructor; when using this, call load() separately to initialise this class. - * - * @see #load(String) - */ - public JoinCostFeatures() { - } - - /** - * Constructor which read a Mary Join Cost file. - */ - public JoinCostFeatures(String fileName) throws IOException, MaryConfigurationException { - load(fileName, null, null, (float) 0.5); - } - - /** - * Initialise this join cost function by reading the appropriate settings from the MaryProperties using the given - * configPrefix. - * - * @param configPrefix - * the prefix for the (voice-specific) config entries to use when looking up files to load. - */ - public void init(String configPrefix) throws MaryConfigurationException { - String joinFileName = MaryProperties.needFilename(configPrefix + ".joinCostFile"); - String precomputedJoinCostFileName = MaryProperties.getFilename(configPrefix + ".precomputedJoinCostFile"); - float wSignal = Float.parseFloat(MaryProperties.getProperty(configPrefix + ".joincostfunction.wSignal", "1.0")); - try { - InputStream joinWeightStream = MaryProperties.getStream(configPrefix + ".joinCostWeights"); - load(joinFileName, joinWeightStream, precomputedJoinCostFileName, wSignal); - } catch (IOException ioe) { - throw new MaryConfigurationException("Problem loading join file " + joinFileName, ioe); - } - } - - /** - * Load weights and values from the given file - * - * @param joinFileName - * the file from which to read default weights and join cost features - * @param weightStream - * an optional file from which to read weights, taking precedence over - * @param precompiledCostFileName - * an optional file containing precompiled join costs - * @param wSignal - * Relative weight of the signal-based join costs relative to the phonetic join costs computed from the target - */ - public void load(String joinFileName, InputStream weightStream, String precompiledCostFileName, float wSignal) - throws IOException, MaryConfigurationException { - loadFromByteBuffer(joinFileName, weightStream, precompiledCostFileName, wSignal); - } - - /** - * Load weights and values from the given file - * - * @param joinFileName - * the file from which to read default weights and join cost features - * @param weightStream - * an optional file from which to read weights, taking precedence over - * @param precompiledCostFileName - * an optional file containing precompiled join costs - * @param wSignal - * Relative weight of the signal-based join costs relative to the phonetic join costs computed from the target - */ - private void loadFromByteBuffer(String joinFileName, InputStream weightStream, String precompiledCostFileName, float wSignal) - throws IOException, MaryConfigurationException { - if (precompiledCostFileName != null) { - precompiledCosts = new PrecompiledJoinCostReader(precompiledCostFileName); - } - this.wSignal = wSignal; - wPhonetic = 1 - wSignal; - /* Open the file */ - FileInputStream fis = new FileInputStream(joinFileName); - FileChannel fc = fis.getChannel(); - ByteBuffer bb = fc.map(FileChannel.MapMode.READ_ONLY, 0, fc.size()); - /* Read the Mary header */ - hdr = new MaryHeader(bb); - if (hdr.getType() != MaryHeader.JOINFEATS) { - throw new IOException("File [" + joinFileName + "] is not a valid Mary join features file."); - } - try { - /* Read the feature weights and feature processors */ - int numberOfFeatures = bb.getInt(); - featureWeight = new float[numberOfFeatures]; - weightFunction = new WeightFunc[numberOfFeatures]; - isLinear = new boolean[numberOfFeatures]; - WeightFunctionManager wfm = new WeightFunctionManager(); - String wfStr = null; - for (int i = 0; i < numberOfFeatures; i++) { - featureWeight[i] = bb.getFloat(); - wfStr = StreamUtils.readUTF(bb); - if ("".equals(wfStr)) - weightFunction[i] = wfm.getWeightFunction("linear"); - else - weightFunction[i] = wfm.getWeightFunction(wfStr); - } - // Overwrite weights and weight functions from file? - if (weightStream != null) { - MaryUtils.getLogger("JoinCostFeatures").debug("Overwriting join cost weights"); - Object[] weightData = readJoinCostWeightsStream(weightStream); - featureWeight = (float[]) weightData[0]; - String[] wf = (String[]) weightData[1]; - if (featureWeight.length != numberOfFeatures) - throw new IllegalArgumentException("Join cost file contains " + numberOfFeatures - + " features, but weight file contains " + featureWeight.length + " feature weights!"); - for (int i = 0; i < numberOfFeatures; i++) { - weightFunction[i] = wfm.getWeightFunction(wf[i]); - } - } - for (int i = 0; i < numberOfFeatures; i++) { - isLinear[i] = weightFunction[i].whoAmI().equals("linear"); - } - - /* Read the left and right Join Cost Features */ - int numberOfUnits = bb.getInt(); - FloatBuffer fb = bb.asFloatBuffer(); - leftJCF = new float[numberOfUnits][]; - rightJCF = new float[numberOfUnits][]; - for (int i = 0; i < numberOfUnits; i++) { - // System.out.println("Reading join features for unit "+i+" out of "+numberOfUnits); - leftJCF[i] = new float[numberOfFeatures]; - fb.get(leftJCF[i]); - rightJCF[i] = new float[numberOfFeatures]; - fb.get(rightJCF[i]); - } - } catch (EOFException e) { - IOException ioe = new IOException("The currently read Join Cost File has prematurely reached EOF."); - ioe.initCause(e); - throw ioe; - - } - if (MaryProperties.getBoolean("debug.show.cost.graph")) { - debugShowCostGraph = true; - cumulWeightedSignalCosts = new double[featureWeight.length]; - jcr = new JoinCostReporter(cumulWeightedSignalCosts); - jcr.showInJFrame("Average signal join costs", false, false); - jcr.start(); - } - - } - - /** - * Load weights and values from the given file - * - * @param joinFileName - * the file from which to read default weights and join cost features - * @param weightStream - * an optional file from which to read weights, taking precedence over - * @param precompiledCostFileName - * an optional file containing precompiled join costs - * @param wSignal - * Relative weight of the signal-based join costs relative to the phonetic join costs computed from the target - */ - private void loadFromStream(String joinFileName, InputStream weightStream, String precompiledCostFileName, float wSignal) - throws IOException, MaryConfigurationException { - if (precompiledCostFileName != null) { - precompiledCosts = new PrecompiledJoinCostReader(precompiledCostFileName); - } - this.wSignal = wSignal; - wPhonetic = 1 - wSignal; - /* Open the file */ - File fid = new File(joinFileName); - DataInput raf = new DataInputStream(new BufferedInputStream(new FileInputStream(fid))); - /* Read the Mary header */ - hdr = new MaryHeader(raf); - if (hdr.getType() != MaryHeader.JOINFEATS) { - throw new MaryConfigurationException("File [" + joinFileName + "] is not a valid Mary join features file."); - } - try { - /* Read the feature weights and feature processors */ - int numberOfFeatures = raf.readInt(); - featureWeight = new float[numberOfFeatures]; - weightFunction = new WeightFunc[numberOfFeatures]; - isLinear = new boolean[numberOfFeatures]; - WeightFunctionManager wfm = new WeightFunctionManager(); - String wfStr = null; - for (int i = 0; i < numberOfFeatures; i++) { - featureWeight[i] = raf.readFloat(); - wfStr = raf.readUTF(); - if ("".equals(wfStr)) - weightFunction[i] = wfm.getWeightFunction("linear"); - else - weightFunction[i] = wfm.getWeightFunction(wfStr); - } - // Overwrite weights and weight functions from file? - if (weightStream != null) { - MaryUtils.getLogger("JoinCostFeatures").debug("Overwriting join cost weights"); - Object[] weightData = readJoinCostWeightsStream(weightStream); - featureWeight = (float[]) weightData[0]; - String[] wf = (String[]) weightData[1]; - if (featureWeight.length != numberOfFeatures) - throw new IllegalArgumentException("Join cost file contains " + numberOfFeatures - + " features, but weight file contains " + featureWeight.length + " feature weights!"); - for (int i = 0; i < numberOfFeatures; i++) { - weightFunction[i] = wfm.getWeightFunction(wf[i]); - } - } - for (int i = 0; i < numberOfFeatures; i++) { - isLinear[i] = weightFunction[i].whoAmI().equals("linear"); - } - - /* Read the left and right Join Cost Features */ - int numberOfUnits = raf.readInt(); - leftJCF = new float[numberOfUnits][]; - rightJCF = new float[numberOfUnits][]; - for (int i = 0; i < numberOfUnits; i++) { - // System.out.println("Reading join features for unit "+i+" out of "+numberOfUnits); - leftJCF[i] = new float[numberOfFeatures]; - for (int j = 0; j < numberOfFeatures; j++) { - leftJCF[i][j] = raf.readFloat(); - } - rightJCF[i] = new float[numberOfFeatures]; - for (int j = 0; j < numberOfFeatures; j++) { - rightJCF[i][j] = raf.readFloat(); - } - } - } catch (EOFException e) { - IOException ioe = new IOException("The currently read Join Cost File has prematurely reached EOF."); - ioe.initCause(e); - throw ioe; - - } - if (MaryProperties.getBoolean("debug.show.cost.graph")) { - debugShowCostGraph = true; - cumulWeightedSignalCosts = new double[featureWeight.length]; - jcr = new JoinCostReporter(cumulWeightedSignalCosts); - jcr.showInJFrame("Average signal join costs", false, false); - jcr.start(); - } - - } - - /** - * Read the join cost weight specifications from the given file. The weights will be normalized such that they sum to one. - * - * @param fileName - * the text file containing the join weights - * */ - public static Object[] readJoinCostWeightsFile(String fileName) throws IOException, FileNotFoundException { - return readJoinCostWeightsStream(new FileInputStream(fileName)); - } - - /** - * Read the join cost weight specifications from the given file. The weights will be normalized such that they sum to one. - * - * @param fileName - * the text file containing the join weights - * */ - public static Object[] readJoinCostWeightsStream(InputStream weightStream) throws IOException, FileNotFoundException { - Vector v = new Vector(16, 16); - Vector vf = new Vector(16, 16); - /* Open the file */ - BufferedReader in = new BufferedReader(new InputStreamReader(weightStream, "UTF-8")); - /* Loop through the lines */ - String line = null; - String[] fields = null; - float sumOfWeights = 0; - while ((line = in.readLine()) != null) { - // System.out.println( line ); - line = line.split("#", 2)[0]; // Remove possible trailing comments - line = line.trim(); // Remove leading and trailing blanks - if (line.equals("")) - continue; // Empty line: don't parse - line = line.split(":", 2)[1].trim(); // Remove the line number and : - // System.out.print( "CLEANED: [" + line + "]" ); - fields = line.split("\\s", 2); // Separate the weight value from the function name - float aWeight = Float.parseFloat(fields[0]); - sumOfWeights += aWeight; - v.add(new Float(aWeight)); // Push the weight - vf.add(fields[1]); // Push the function - // System.out.println( "NBFEA=" + numberOfFeatures ); - } - in.close(); - // System.out.flush(); - /* Export the vector of weighting function names as a String array: */ - String[] wfun = (String[]) vf.toArray(new String[vf.size()]); - /* - * For the weights, create a float array containing the weights, normalized such that they sum to one: - */ - float[] fw = new float[v.size()]; - for (int i = 0; i < fw.length; i++) { - Float aWeight = (Float) v.get(i); - fw[i] = aWeight.floatValue() / sumOfWeights; - } - /* Return these as an Object[2]. */ - return new Object[] { fw, wfun }; - } - - /*****************/ - /* ACCESSORS */ - /*****************/ - - /** - * Get the number of feature weights and weighting functions. - */ - public int getNumberOfFeatures() { - return (featureWeight.length); - } - - /** - * Get the number of units. - */ - public int getNumberOfUnits() { - return (leftJCF.length); - } - - /** - * Gets the array of left join cost features for a particular unit index. - * - * @param u - * The index of the considered unit. - * - * @return The array of left join cost features for the given unit. - */ - public float[] getLeftJCF(int u) { - if (u < 0) { - throw new RuntimeException("The unit index [" + u + "] is out of range: a unit index can't be negative."); - } - if (u > getNumberOfUnits()) { - throw new RuntimeException("The unit index [" + u + "] is out of range: this file contains [" + getNumberOfUnits() - + "] units."); - } - return (leftJCF[u]); - } - - /** - * Gets the array of right join cost features for a particular unit index. - * - * @param u - * The index of the considered unit. - * - * @return The array of right join cost features for the given unit. - */ - public float[] getRightJCF(int u) { - if (u < 0) { - throw new RuntimeException("The unit index [" + u + "] is out of range: a unit index can't be negative."); - } - if (u > getNumberOfUnits()) { - throw new RuntimeException("The unit index [" + u + "] is out of range: this file contains [" + getNumberOfUnits() - + "] units."); - } - return (rightJCF[u]); - } - - /*****************/ - /* MISC METHODS */ - /*****************/ - - /** - * Deliver the join cost between two units described by their index. - * - * @param u1 - * the left unit - * @param u2 - * the right unit - * - * @return the cost of joining the right Join Cost features of the left unit with the left Join Cost Features of the right - * unit. - */ - public double cost(int u1, int u2) { - /* Check the given indexes */ - if (u1 < 0) { - throw new RuntimeException("The left unit index [" + u1 + "] is out of range: a unit index can't be negative."); - } - // if ( u1 > getNumberOfUnits() ) { - if (u1 > leftJCF.length) { - throw new RuntimeException("The left unit index [" + u1 + "] is out of range: this file contains [" - + getNumberOfUnits() + "] units."); - } - if (u2 < 0) { - throw new RuntimeException("The right unit index [" + u2 + "] is out of range: a unit index can't be negative."); - } - // if ( u2 > getNumberOfUnits() ) { - if (u2 > leftJCF.length) { - throw new RuntimeException("The right unit index [" + u2 + "] is out of range: this file contains [" - + getNumberOfUnits() + "] units."); - } - if (debugShowCostGraph) { - jcr.tick(); - } - /* Cumulate the join costs for each feature */ - double res = 0.0; - float[] v1 = rightJCF[u1]; - float[] v2 = leftJCF[u2]; - for (int i = 0; i < v1.length; i++) { - float a = v1[i]; - float b = v2[i]; - // if (!Float.isNaN(v1[i]) && !Float.isNaN(v2[i])) { - if (!(a != a) && !(b != b)) { - double c; - if (isLinear[i]) { - c = featureWeight[i] * (a > b ? (a - b) : (b - a)); - } else { - c = featureWeight[i] * weightFunction[i].cost(a, b); - } - res += c; - if (debugShowCostGraph) { - cumulWeightedSignalCosts[i] += wSignal * c; - } - } // if anything is NaN, count the cost as 0. - } - return (res); - } - - /** - * A combined cost computation, as a weighted sum of the signal-based cost (computed from the units) and the phonetics-based - * cost (computed from the targets). - * - * @param t1 - * The left target. - * @param u1 - * The left unit. - * @param t2 - * The right target. - * @param u2 - * The right unit. - * - * @return the cost of joining the left unit with the right unit, as a non-negative value. - */ - public double cost(Target t1, Unit u1, Target t2, Unit u2) { - // Units of length 0 cannot be joined: - if (u1.duration == 0 || u2.duration == 0) - return Double.POSITIVE_INFINITY; - // In the case of diphones, replace them with the relevant part: - boolean bothDiphones = true; - if (u1 instanceof DiphoneUnit) { - u1 = ((DiphoneUnit) u1).right; - } else { - bothDiphones = false; - } - if (u2 instanceof DiphoneUnit) { - u2 = ((DiphoneUnit) u2).left; - } else { - bothDiphones = false; - } - - if (u1.index + 1 == u2.index) - return 0; - // Either not half phone synthesis, or at a diphone boundary - double cost = 1; // basic penalty for joins of non-contiguous units. - if (bothDiphones && precompiledCosts != null) { - cost += precompiledCosts.cost(t1, u1, t2, u2); - } else { // need to actually compute the cost - cost += cost(u1.index, u2.index); - } - return cost; - } - - /** - * A phonetic join cost, computed solely from the target. - * - * @param t1 - * the left target - * @param t2 - * the right target - * @return a non-negative join cost, usually between 0 (best) and 1 (worst). - * @deprecated - */ - protected double cost(Target t1, Target t2) { - // TODO: This is really ad hoc for the moment. Redo once we know what we are doing. - // Add penalties for a number of criteria. - double cost = 0; - ByteValuedFeatureProcessor stressProcessor = new MaryGenericFeatureProcessors.Stressed("", - new MaryGenericFeatureProcessors.SyllableNavigator()); - // Stressed? - boolean stressed1 = stressProcessor.process(t1) == (byte) 1; - boolean stressed2 = stressProcessor.process(t1) == (byte) 1; - // Try to avoid joining in a stressed syllable: - if (stressed1 || stressed2) - cost += 0.2; - Allophone p1 = t1.getAllophone(); - Allophone p2 = t2.getAllophone(); - - // Discourage joining vowels: - if (p1.isVowel() || p2.isVowel()) - cost += 0.2; - // Discourage joining glides: - if (p1.isGlide() || p2.isGlide()) - cost += 0.2; - // Discourage joining voiced segments: - if (p1.isVoiced() || p2.isVoiced()) - cost += 0.1; - // If both are voiced, it's really bad - if (p1.isVoiced() && p2.isVoiced()) - cost += 0.1; - // Slightly penalize nasals and liquids - if (p1.isNasal() || p2.isNasal()) - cost += 0.05; - if (p1.isLiquid() || p2.isLiquid()) - cost += 0.05; - // Fricatives -- nothing? - // Plosives -- nothing? - - if (cost > 1) - cost = 1; - return cost; - } - - public static class JoinCostReporter extends Histogram { - private double[] data; - private int lastN = 0; - private int nCostComputations = 0; - - public JoinCostReporter(double[] data) { - super(0, 1, data); - this.data = data; - } - - public void start() { - new Thread() { - public void run() { - while (isVisible()) { - try { - Thread.sleep(500); - } catch (InterruptedException ie) { - } - updateGraph(); - } - } - }.start(); - } - - /** - * Register one new cost computation - */ - public void tick() { - nCostComputations++; - } - - protected void updateGraph() { - if (nCostComputations == lastN) - return; - lastN = nCostComputations; - double[] newCosts = new double[data.length]; - for (int i = 0; i < newCosts.length; i++) { - newCosts[i] = data[i] / nCostComputations; - } - updateData(0, 1, newCosts); - repaint(); - } - } - -} diff --git a/marytts-runtime/src/main/java/marytts/unitselection/select/JoinCostFunction.java b/marytts-runtime/src/main/java/marytts/unitselection/select/JoinCostFunction.java deleted file mode 100644 index 2bc4f38ac1..0000000000 --- a/marytts-runtime/src/main/java/marytts/unitselection/select/JoinCostFunction.java +++ /dev/null @@ -1,77 +0,0 @@ -/** - * Copyright 2006 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.unitselection.select; - -import java.io.IOException; -import java.io.InputStream; - -import marytts.exceptions.MaryConfigurationException; -import marytts.unitselection.data.Unit; - -/** - * A join cost function for evaluating the goodness-of-fit of a given pair of left and right unit. - * - * @author Marc Schröder - * - */ -public interface JoinCostFunction { - /** - * Compute the goodness-of-fit of joining two units, given the corresponding targets - * - * @param t1 - * the left target - * @param u1 - * the proposed left unit - * @param t2 - * the right target - * @param u3 - * the proposed right unit - * @return a non-negative number; smaller values mean better fit, i.e. smaller cost. - */ - public double cost(Target t1, Unit u1, Target t2, Unit u2); - - /** - * Initialise this join cost function by reading the appropriate settings from the MaryProperties using the given - * configPrefix. - * - * @param configPrefix - * the prefix for the (voice-specific) config entries to use when looking up files to load. - * @throws MaryConfigurationException - * if there is a configuration problem - */ - public void init(String configPrefix) throws MaryConfigurationException; - - /** - * Load weights and values from the given file - * - * @param joinFileName - * the file from which to read default weights and join cost features - * @param weightStream - * an optional file from which to read weights, taking precedence over - * @param precompiledCostFileName - * an optional file containing precompiled join costs - * @param wSignal - * Relative weight of the signal-based join costs relative to the phonetic join costs computed from the target - */ - @Deprecated - public void load(String joinFileName, InputStream weightStream, String precompiledCostFileName, float wSignal) - throws IOException, MaryConfigurationException; - -} diff --git a/marytts-runtime/src/main/java/marytts/unitselection/select/JoinModelCost.java b/marytts-runtime/src/main/java/marytts/unitselection/select/JoinModelCost.java deleted file mode 100644 index 8a14c1ecb9..0000000000 --- a/marytts-runtime/src/main/java/marytts/unitselection/select/JoinModelCost.java +++ /dev/null @@ -1,217 +0,0 @@ -/** - * Copyright 2006 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.unitselection.select; - -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; - -import marytts.cart.CART; -import marytts.cart.Node; -import marytts.cart.LeafNode.PdfLeafNode; -import marytts.cart.io.HTSCARTReader; -import marytts.exceptions.MaryConfigurationException; -import marytts.features.FeatureDefinition; -import marytts.features.FeatureVector; -import marytts.htsengine.PhoneTranslator; -import marytts.htsengine.HMMData.PdfFileFormat; -import marytts.server.MaryProperties; -import marytts.signalproc.analysis.distance.DistanceComputer; -import marytts.unitselection.data.DiphoneUnit; -import marytts.unitselection.data.Unit; - -public class JoinModelCost implements JoinCostFunction { - protected int nCostComputations = 0; - - /****************/ - /* DATA FIELDS */ - /****************/ - private JoinCostFeatures jcf = null; - - CART[] joinTree = null; // an array of carts, one per HMM state. - - private float f0Weight; - - private FeatureDefinition featureDef = null; - - private boolean debugShowCostGraph = false; - - /****************/ - /* CONSTRUCTORS */ - /****************/ - - /** - * Empty constructor; when using this, call load() separately to initialise this class. - * - * @see #load(String) - */ - public JoinModelCost() { - } - - /** - * Initialise this join cost function by reading the appropriate settings from the MaryProperties using the given - * configPrefix. - * - * @param configPrefix - * the prefix for the (voice-specific) config entries to use when looking up files to load. - */ - public void init(String configPrefix) throws MaryConfigurationException { - try { - String joinFileName = MaryProperties.needFilename(configPrefix + ".joinCostFile"); - InputStream joinPdfStream = MaryProperties.needStream(configPrefix + ".joinPdfFile"); - InputStream joinTreeStream = MaryProperties.needStream(configPrefix + ".joinTreeFile"); - // CHECK not tested the trickyPhonesFile needs to be added into the configuration file - String trickyPhonesFileName = MaryProperties.needFilename(configPrefix + ".trickyPhonesFile"); - load(joinFileName, joinPdfStream, joinTreeStream, trickyPhonesFileName); - } catch (IOException ioe) { - throw new MaryConfigurationException("Problem loading join file", ioe); - } - } - - @Override - @Deprecated - public void load(String a, InputStream b, String c, float d) { - throw new RuntimeException("Do not use load() -- use init()"); - } - - /** - * Load weights and values from the given file - * - * @param joinFileName - * the file from which to read join cost features - * @param joinPdfFileName - * the file from which to read the Gaussian models in the leaves of the tree - * @param joinTreeFileName - * the file from which to read the Tree, in HTS format. - */ - public void load(String joinFileName, InputStream joinPdfStream, InputStream joinTreeStream, String trickyPhonesFile) - throws IOException, MaryConfigurationException { - jcf = new JoinCostFeatures(joinFileName); - - assert featureDef != null : "Expected to have a feature definition, but it is null!"; - - /* Load Trees */ - HTSCARTReader htsReader = new HTSCARTReader(); - int numStates = 1; // just one state in the joinModeller - - // Check if there are tricky phones, and create a PhoneTranslator object - PhoneTranslator phTranslator = new PhoneTranslator(new FileInputStream(trickyPhonesFile)); - - try { - // joinTree.loadTreeSetGeneral(joinTreeFileName, 0, featureDef); - joinTree = htsReader.load(numStates, joinTreeStream, joinPdfStream, PdfFileFormat.join, featureDef, phTranslator); - - } catch (Exception e) { - IOException ioe = new IOException("Cannot load join model trees"); - ioe.initCause(e); - throw ioe; - } - - } - - /** - * Set the feature definition to use for interpreting target feature vectors. - * - * @param def - * the feature definition to use. - */ - public void setFeatureDefinition(FeatureDefinition def) { - this.featureDef = def; - } - - /*****************/ - /* MISC METHODS */ - /*****************/ - - /** - * A combined cost computation, as a weighted sum of the signal-based cost (computed from the units) and the phonetics-based - * cost (computed from the targets). - * - * @param t1 - * The left target. - * @param u1 - * The left unit. - * @param t2 - * The right target. - * @param u2 - * The right unit. - * - * @return the cost of joining the left unit with the right unit, as a non-negative value. - */ - public double cost(Target t1, Unit u1, Target t2, Unit u2) { - // Units of length 0 cannot be joined: - if (u1.duration == 0 || u2.duration == 0) - return Double.POSITIVE_INFINITY; - // In the case of diphones, replace them with the relevant part: - if (u1 instanceof DiphoneUnit) { - u1 = ((DiphoneUnit) u1).right; - } - if (u2 instanceof DiphoneUnit) { - u2 = ((DiphoneUnit) u2).left; - } - - if (u1.index + 1 == u2.index) - return 0; - double cost = 1; // basic penalty for joins of non-contiguous units. - - float[] v1 = jcf.getRightJCF(u1.index); - float[] v2 = jcf.getLeftJCF(u2.index); - // double[] diff = new double[v1.length]; - // for ( int i = 0; i < v1.length; i++ ) { - // diff[i] = (double)v1[i] - v2[i]; - // } - double[] diff = new double[v1.length]; - for (int i = 0; i < v1.length; i++) { - diff[i] = (double) v1[i] - v2[i]; - } - - // Now evaluate likelihood of the diff under the join model - // Compute the model name: - assert featureDef != null : "Feature Definition was not set"; - FeatureVector fv1 = null; - if (t1 instanceof DiphoneTarget) { - HalfPhoneTarget hpt1 = ((DiphoneTarget) t1).right; - assert hpt1 != null; - fv1 = hpt1.getFeatureVector(); - } else { - fv1 = t1.getFeatureVector(); - } - assert fv1 != null : "Target has no feature vector"; - // String modelName = contextTranslator.features2context(featureDef, fv1, featureList); - - int state = 0; // just one state in the joinModeller - double[] mean; - double[] variance; - - Node node = joinTree[state].interpretToNode(fv1, 1); - - assert node instanceof PdfLeafNode : "The node must be a PdfLeafNode."; - - mean = ((PdfLeafNode) node).getMean(); - variance = ((PdfLeafNode) node).getVariance(); - - double distance = DistanceComputer.getNormalizedEuclideanDistance(diff, mean, variance); - - cost += distance; - - return cost; - } - -} diff --git a/marytts-runtime/src/main/java/marytts/unitselection/select/PrecompiledJoinCostReader.java b/marytts-runtime/src/main/java/marytts/unitselection/select/PrecompiledJoinCostReader.java deleted file mode 100644 index ff8b48a890..0000000000 --- a/marytts-runtime/src/main/java/marytts/unitselection/select/PrecompiledJoinCostReader.java +++ /dev/null @@ -1,156 +0,0 @@ -/** - * Portions Copyright 2006 DFKI GmbH. - * Portions Copyright 2001 Sun Microsystems, Inc. - * Portions Copyright 1999-2001 Language Technologies Institute, - * Carnegie Mellon University. - * All Rights Reserved. Use is subject to license terms. - * - * Permission is hereby granted, free of charge, to use and distribute - * this software and its documentation without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of this work, and to - * permit persons to whom this work is furnished to do so, subject to - * the following conditions: - * - * 1. The code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * 2. Any modifications must be clearly marked as such. - * 3. Original authors' names are not deleted. - * 4. The authors' names are not used to endorse or promote products - * derived from this software without specific prior written - * permission. - * - * DFKI GMBH AND THE CONTRIBUTORS TO THIS WORK DISCLAIM ALL WARRANTIES WITH - * REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL DFKI GMBH NOR THE - * CONTRIBUTORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL - * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR - * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS - * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF - * THIS SOFTWARE. - */ -package marytts.unitselection.select; - -import java.io.BufferedInputStream; -import java.io.DataInputStream; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.util.HashMap; -import java.util.Map; - -import marytts.exceptions.MaryConfigurationException; -import marytts.server.MaryProperties; -import marytts.unitselection.data.Unit; -import marytts.util.data.MaryHeader; - -/** - * Loads a precompiled join cost file and provides access to the join cost. - * - */ -public class PrecompiledJoinCostReader implements JoinCostFunction { - - private MaryHeader hdr = null; - - // keys = Integers representing left unit index; - // values = maps containing - // - keys = Integers representing right unit index; - // - values = Floats representing the jost of joining them. - protected Map left; - - /** - * Empty constructor; need to call load() separately. - * - * @see #load(String) - */ - public PrecompiledJoinCostReader() { - } - - /** - * Create a precompiled join cost file reader from the given file - * - * @param fileName - * the file to read - * @throws IOException - * if a problem occurs while reading - */ - public PrecompiledJoinCostReader(String fileName) throws IOException, MaryConfigurationException { - load(fileName, null, null, 0); - } - - /** - * Initialise this join cost function by reading the appropriate settings from the MaryProperties using the given - * configPrefix. - * - * @param configPrefix - * the prefix for the (voice-specific) config entries to use when looking up files to load. - */ - public void init(String configPrefix) throws MaryConfigurationException { - String precomputedJoinCostFileName = MaryProperties.getFilename(configPrefix + ".precomputedJoinCostFile"); - try { - load(precomputedJoinCostFileName, null, null, 0); - } catch (IOException ioe) { - throw new MaryConfigurationException("Problem loading join file " + precomputedJoinCostFileName, ioe); - } - } - - /** - * Load the given precompiled join cost file - * - * @param fileName - * the file to read - * @param dummy - * not used, just used to fulfil the join cost function interface - * @param dummy2 - * not used, just used to fulfil the join cost function interface - * @throws IOException - * if a problem occurs while reading - */ - @Override - public void load(String fileName, InputStream dummy, String dummy2, float dummy3) throws IOException, - MaryConfigurationException { - /* Open the file */ - DataInputStream dis = new DataInputStream(new BufferedInputStream(new FileInputStream(fileName))); - hdr = new MaryHeader(dis); - if (hdr.getType() != MaryHeader.PRECOMPUTED_JOINCOSTS) { - throw new MaryConfigurationException("File [" + fileName + "] is not a valid Mary precompiled join costs file."); - } - /* Read the number of units */ - int numberOfLeftUnits = dis.readInt(); - if (numberOfLeftUnits < 0) { - throw new MaryConfigurationException("File [" + fileName + "] has a negative number of units. Aborting."); - } - - left = new HashMap(); - /* Read the start times and durations */ - for (int i = 0; i < numberOfLeftUnits; i++) { - int leftIndex = dis.readInt(); - int numberOfRightUnits = dis.readInt(); - Map right = new HashMap(); - left.put(new Integer(leftIndex), right); - for (int j = 0; j < numberOfRightUnits; j++) { - int rightIndex = dis.readInt(); - float cost = dis.readFloat(); - right.put(new Integer(rightIndex), new Float(cost)); - } - } - - } - - /** - * Return the (precomputed) cost of joining the two given units; if there is no precomputed cost, return - * Double.POSITIVE_INFINITY. - */ - public double cost(Target t1, Unit uleft, Target t2, Unit uright) { - Integer leftIndex = new Integer(uleft.index); - Map rightUnitsMap = (Map) left.get(leftIndex); - if (rightUnitsMap == null) - return Double.POSITIVE_INFINITY; - Integer rightIndex = new Integer(uright.index); - Float cost = (Float) rightUnitsMap.get(rightIndex); - if (cost == null) - return Double.POSITIVE_INFINITY; - return cost.doubleValue(); - } - -} diff --git a/marytts-runtime/src/main/java/marytts/unitselection/select/SelectedUnit.java b/marytts-runtime/src/main/java/marytts/unitselection/select/SelectedUnit.java deleted file mode 100644 index 2e42c189b0..0000000000 --- a/marytts-runtime/src/main/java/marytts/unitselection/select/SelectedUnit.java +++ /dev/null @@ -1,76 +0,0 @@ -/** - * Copyright 2006 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.unitselection.select; - -import marytts.unitselection.data.Unit; - -/** - * A unit selected from Viterbi - * - * @author Marc Schröder - * - */ -public class SelectedUnit { - protected Unit unit; - protected Target target; - protected Object concatenationData; - protected double[] audio; - - public SelectedUnit(Unit unit, Target target) { - this.unit = unit; - this.target = target; - this.audio = null; - } - - public Unit getUnit() { - return unit; - } - - public Target getTarget() { - return target; - } - - /** - * Remember data about this selected unit which is relevant for unit concatenation. What type of data is saved here depends on - * the UnitConcatenator used. - * - * @param concatenationData - */ - public void setConcatenationData(Object concatenationData) { - this.concatenationData = concatenationData; - } - - public Object getConcatenationData() { - return concatenationData; - } - - public void setAudio(double[] audio) { - this.audio = audio; - } - - public double[] getAudio() { - return audio; - } - - public String toString() { - return "Target: " + target.toString() + " Unit: " + unit.toString() + " target duration " - + target.getTargetDurationInSeconds(); - } -} diff --git a/marytts-runtime/src/main/java/marytts/unitselection/select/StatisticalCostFunction.java b/marytts-runtime/src/main/java/marytts/unitselection/select/StatisticalCostFunction.java deleted file mode 100644 index a322568954..0000000000 --- a/marytts-runtime/src/main/java/marytts/unitselection/select/StatisticalCostFunction.java +++ /dev/null @@ -1,44 +0,0 @@ -/** - * Copyright 2006 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.unitselection.select; - -import marytts.exceptions.MaryConfigurationException; -import marytts.unitselection.data.Unit; - -/** - * A statistical cost function - * - * @author Sathish Pammi - * - */ -public interface StatisticalCostFunction { - - public double cost(Unit u1, Unit u2); - - /** - * Initialise this scost cost function by reading the appropriate settings from the MaryProperties using the given - * configPrefix. - * - * @param configPrefix - * the prefix for the (voice-specific) config entries to use when looking up files to load. - */ - public void init(String configPrefix) throws MaryConfigurationException; - -} diff --git a/marytts-runtime/src/main/java/marytts/unitselection/select/StatisticalModelCost.java b/marytts-runtime/src/main/java/marytts/unitselection/select/StatisticalModelCost.java deleted file mode 100644 index 069a533c99..0000000000 --- a/marytts-runtime/src/main/java/marytts/unitselection/select/StatisticalModelCost.java +++ /dev/null @@ -1,143 +0,0 @@ -/** - * Copyright 2006 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.unitselection.select; - -import marytts.exceptions.MaryConfigurationException; -import marytts.server.MaryProperties; -import marytts.unitselection.data.DiphoneUnit; -import marytts.unitselection.data.SCostFileReader; -import marytts.unitselection.data.Unit; -import marytts.util.data.MaryHeader; - -/** - * StatisticalModelCost for a given unit - * - * @author sathish pammi - * - */ -public class StatisticalModelCost implements StatisticalCostFunction { - - protected SCostFileReader sCostReader; - protected float sCostWeight; - - /****************/ - /* DATA FIELDS */ - /****************/ - private MaryHeader hdr = null; - - /****************/ - /* CONSTRUCTORS */ - /****************/ - - /** - * Empty constructor; when using this, call init() separately to initialise this class. - * - * @see #init(String) - */ - public StatisticalModelCost() { - } - - /** - * Initialise this scost function by reading the appropriate settings from the MaryProperties using the given configPrefix. - * - * @param configPrefix - * the prefix for the (voice-specific) config entries to use when looking up files to load. - */ - public void init(String configPrefix) throws MaryConfigurationException { - try { - String sCostFileName = MaryProperties.needFilename(configPrefix + ".sCostFile"); - sCostWeight = Float.parseFloat(MaryProperties.getProperty(configPrefix + ".sCostWeight", "1.0")); - sCostReader = new SCostFileReader(sCostFileName); - } catch (Exception e) { - throw new MaryConfigurationException("Cannot initialise scost model", e); - } - } - - /*****************/ - /* ACCESSORS */ - /*****************/ - - /** - * Get the number of units. - */ - public int getNumberOfUnits() { - return (sCostReader.getNumberOfUnits()); - } - - /*****************/ - /* MISC METHODS */ - /*****************/ - - public double cost(Unit u1, Unit u2) { - - // Units of length 0 cannot be joined: - if (u1.duration == 0 || u2.duration == 0) - return Double.POSITIVE_INFINITY; - // In the case of diphones, replace them with the relevant part: - if (u1 instanceof DiphoneUnit) { - u1 = ((DiphoneUnit) u1).right; - } - if (u2 instanceof DiphoneUnit) { - u2 = ((DiphoneUnit) u2).left; - } - - /** - * TODO uncomment below line to make ---> cost of successive units = 0 - */ - // if (u1.getIndex()+1 == u2.getIndex()) return 0; - - double sCost1 = sCostReader.getSCost(u1.index); - double sCost2 = sCostReader.getSCost(u2.index); - - return ((sCost1 + sCost2) / 2.0); - } - - /** - * Cost function for a given units if consecutive == true, and if they are consecutive units make cost = 0 - * - * @param u1 - * @param u2 - * @param consecutive - * @return - */ - public double cost(Unit u1, Unit u2, boolean consecutive) { - - // Units of length 0 cannot be joined: - if (u1.duration == 0 || u2.duration == 0) - return Double.POSITIVE_INFINITY; - // In the case of diphones, replace them with the relevant part: - if (u1 instanceof DiphoneUnit) { - u1 = ((DiphoneUnit) u1).right; - } - if (u2 instanceof DiphoneUnit) { - u2 = ((DiphoneUnit) u2).left; - } - - // if consecutive == true, and if they are consecutive units make cost = 0 - if (consecutive && (u1.index + 1 == u2.index)) - return 0; - - double sCost1 = sCostReader.getSCost(u1.index); - double sCost2 = sCostReader.getSCost(u2.index); - - return ((sCost1 + sCost2) / 2.0); - } - -} diff --git a/marytts-runtime/src/main/java/marytts/unitselection/select/Target.java b/marytts-runtime/src/main/java/marytts/unitselection/select/Target.java deleted file mode 100644 index ce9b2d596b..0000000000 --- a/marytts-runtime/src/main/java/marytts/unitselection/select/Target.java +++ /dev/null @@ -1,191 +0,0 @@ -/** - * Copyright 2006 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.unitselection.select; - -import marytts.datatypes.MaryXML; -import marytts.features.FeatureVector; -import marytts.features.MaryGenericFeatureProcessors; -import marytts.modules.phonemiser.Allophone; -import marytts.modules.phonemiser.AllophoneSet; -import marytts.modules.synthesis.Voice; -import marytts.util.MaryRuntimeUtils; -import marytts.util.dom.MaryDomUtils; - -import org.w3c.dom.Element; -import org.w3c.dom.Node; -import org.w3c.dom.UserDataHandler; - -/** - * A representation of a target representing the ideal properties of a unit in a target utterance. - * - * @author Marc Schröder - * - */ -public class Target { - protected String name; - protected Element maryxmlElement; - - protected FeatureVector featureVector = null; - - protected float duration = -1; - protected float f0 = -1; - protected int isSilence = -1; - - /** - * Create a target associated to the given element in the MaryXML tree. - * - * @param name - * a name for the target, which may or may not coincide with the segment name. - * @param maryxmlElement - * the phone or boundary element in the MaryXML tree to be associated with this target. - */ - public Target(String name, Element maryxmlElement) { - this.name = name; - this.maryxmlElement = maryxmlElement; - } - - public Element getMaryxmlElement() { - return maryxmlElement; - } - - public String getName() { - return name; - } - - public FeatureVector getFeatureVector() { - return featureVector; - } - - public void setFeatureVector(FeatureVector featureVector) { - this.featureVector = featureVector; - } - - public float getTargetDurationInSeconds() { - if (duration != -1) { - return duration; - } else { - if (maryxmlElement == null) - return 0; - // throw new NullPointerException("Target "+name+" does not have a maryxml element."); - duration = new MaryGenericFeatureProcessors.UnitDuration().process(this); - return duration; - } - } - - /** - * adapted from {@link MaryGenericFeatureProcessors.UnitDuration#process} - * - * @param newDuration - */ - public void setTargetDurationInSeconds(float newDuration) { - if (maryxmlElement != null) { - if (maryxmlElement.getTagName().equals(MaryXML.PHONE)) { - maryxmlElement.setAttribute("d", Float.toString(newDuration)); - } else { - assert maryxmlElement.getTagName().equals(MaryXML.BOUNDARY) : "segment should be a phone or a boundary, but is a " - + maryxmlElement.getTagName(); - maryxmlElement.setAttribute("duration", Float.toString(newDuration)); - } - } - } - - public float getTargetF0InHz() { - if (f0 != -1) { - return f0; - } else { - if (maryxmlElement == null) - throw new NullPointerException("Target " + name + " does not have a maryxml element."); - float logf0 = new MaryGenericFeatureProcessors.UnitLogF0().process(this); - if (logf0 == 0) - f0 = 0; - else - f0 = (float) Math.exp(logf0); - return f0; - } - } - - public boolean hasFeatureVector() { - return featureVector != null; - } - - public static UserDataHandler targetFeatureCloner = new UserDataHandler() { - public void handle(short operation, String key, Object data, Node src, Node dest) { - if (operation == UserDataHandler.NODE_CLONED && key == "target") { - dest.setUserData(key, data, this); - System.err.println("yay"); - } else { - System.err.println("nay"); - } - } - }; - - /** - * Determine whether this target is a silence target - * - * @return true if the target represents silence, false otherwise - */ - public boolean isSilence() { - - if (isSilence == -1) { - // TODO: how do we know the silence symbol here? - String silenceSymbol = "_"; - if (name.startsWith(silenceSymbol)) { - isSilence = 1; // true - } else { - isSilence = 0; // false - } - } - return isSilence == 1; - } - - public Allophone getAllophone() { - if (maryxmlElement != null) { - AllophoneSet allophoneSet = null; - Element voiceElement = (Element) MaryDomUtils.getAncestor(maryxmlElement, MaryXML.VOICE); - if (voiceElement != null) { - Voice v = Voice.getVoice(voiceElement); - if (v != null) { - allophoneSet = v.getAllophoneSet(); - } - } - if (allophoneSet == null) { - try { - allophoneSet = MaryRuntimeUtils.determineAllophoneSet(maryxmlElement); - } catch (Exception e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - } - String sampa; - if (maryxmlElement.getNodeName().equals(MaryXML.PHONE)) { - sampa = maryxmlElement.getAttribute("p"); - } else { - assert maryxmlElement.getNodeName().equals(MaryXML.BOUNDARY); - sampa = "_"; - } - return allophoneSet.getAllophone(sampa); - } - return null; - } - - public String toString() { - return name; - } -} diff --git a/marytts-runtime/src/main/java/marytts/unitselection/select/TargetCostFunction.java b/marytts-runtime/src/main/java/marytts/unitselection/select/TargetCostFunction.java deleted file mode 100644 index 5ca89181d5..0000000000 --- a/marytts-runtime/src/main/java/marytts/unitselection/select/TargetCostFunction.java +++ /dev/null @@ -1,126 +0,0 @@ -/** - * Copyright 2006 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.unitselection.select; - -import java.io.IOException; -import java.io.InputStream; - -import marytts.exceptions.MaryConfigurationException; -import marytts.features.FeatureDefinition; -import marytts.features.FeatureProcessorManager; -import marytts.features.FeatureVector; -import marytts.unitselection.data.FeatureFileReader; -import marytts.unitselection.data.Unit; - -/** - * A target cost function for evaluating the goodness-of-fit of a given unit for a given target. - * - * @author Marc Schröder - * - */ -public interface TargetCostFunction { - /** - * Initialise the data needed to do a target cost computation. - * - * @param featureFileName - * name of a file containing the unit features - * @param weightsStream - * an optional weights file -- if non-null, contains feature weights that override the ones present in the feature - * file. - * @param featProc - * a feature processor manager which can provide feature processors to compute the features for a target at run - * time - * @throws IOException - * @throws MaryConfigurationException - * if a configuration problem is detected while loading the data - */ - public void load(String featureFileName, InputStream weightsStream, FeatureProcessorManager featProc) throws IOException, - MaryConfigurationException; - - /** - * Initialise the data needed to do a target cost computation. - * - * @param featureFileReader - * a reader for the file containing the unit features - * @param weightsFile - * an optional weights file -- if non-null, contains feature weights that override the ones present in the feature - * file. - * @param featProc - * a feature processor manager which can provide feature processors to compute the features for a target at run - * time - * @throws IOException - */ - public void load(FeatureFileReader featureFileReader, InputStream weightsStream, FeatureProcessorManager featProc) - throws IOException; - - /** - * Compute the goodness-of-fit of a given unit for a given target. - * - * @param target - * @param unit - * @return a non-negative number; smaller values mean better fit, i.e. smaller cost. - */ - public double cost(Target target, Unit unit); - - /** - * Compute the features for a given target, and store them in the target. - * - * @param target - * the target for which to compute the features - * @see Target#getFeatureVector() - */ - public void computeTargetFeatures(Target target); - - /** - * Provide access to the Feature Definition used. - * - * @return the feature definition object. - */ - public FeatureDefinition getFeatureDefinition(); - - /** - * Get the string representation of the feature value associated with the given unit - * - * @param unit - * the unit whose feature value is requested - * @param featureName - * name of the feature requested - * @return a string representation of the feature value - * @throws IllegalArgumentException - * if featureName is not a known feature - */ - public String getFeature(Unit unit, String featureName); - - /** - * Get the target cost feature vector for the given unit. - * - * @param unit - * @return - */ - public FeatureVector getFeatureVector(Unit unit); - - /** - * Get all feature vectors. This is useful for more efficient access. - * - * @return the full array of feature vectors, or null if this method is not supported. - */ - public FeatureVector[] getFeatureVectors(); - -} diff --git a/marytts-runtime/src/main/java/marytts/unitselection/select/UnitSelector.java b/marytts-runtime/src/main/java/marytts/unitselection/select/UnitSelector.java deleted file mode 100644 index bfafaa12f8..0000000000 --- a/marytts-runtime/src/main/java/marytts/unitselection/select/UnitSelector.java +++ /dev/null @@ -1,158 +0,0 @@ -/** - * Copyright 2006 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.unitselection.select; - -import java.util.ArrayList; -import java.util.List; - -import marytts.datatypes.MaryXML; -import marytts.exceptions.SynthesisException; -import marytts.unitselection.data.UnitDatabase; -import marytts.unitselection.select.viterbi.Viterbi; -import marytts.util.MaryUtils; - -import org.apache.log4j.Logger; -import org.w3c.dom.Element; -import org.w3c.dom.NodeList; - -/** - * Selects the units for an utterance - * - * @author Marc Schröder - * - */ -public class UnitSelector { - protected UnitDatabase database; - protected Logger logger; - protected float targetCostWeight; - protected float sCostWeight = -1; - protected int beamSize; - - /** - * Initialise the unit selector. Need to call load() separately. - * - * @see #load(UnitDatabase) - */ - public UnitSelector() throws Exception { - logger = MaryUtils.getLogger(this.getClass()); - } - - public void load(UnitDatabase unitDatabase, float targetCostWeight, int beamSize) { - this.database = unitDatabase; - this.targetCostWeight = targetCostWeight; - this.beamSize = beamSize; - } - - public void load(UnitDatabase unitDatabase, float targetCostWeight, float sCostWeight, int beamSize) { - this.database = unitDatabase; - this.targetCostWeight = targetCostWeight; - this.sCostWeight = sCostWeight; - this.beamSize = beamSize; - } - - /** - * Select the units for the targets in the given list of tokens and boundaries. Collect them in a list and return it. - * - * @param tokensAndBoundaries - * the token and boundary MaryXML elements representing an utterance. - * @param voice - * the voice with which to synthesize - * @param db - * the database of the voice - * @param unitNamer - * a unitNamer - * @return a list of SelectedUnit objects - * @throws IllegalStateException - * if no path for generating the target utterance could be found - */ - public List selectUnits(List tokensAndBoundaries, marytts.modules.synthesis.Voice voice) - throws SynthesisException { - long time = System.currentTimeMillis(); - - List segmentsAndBoundaries = new ArrayList(); - for (Element tOrB : tokensAndBoundaries) { - if (tOrB.getTagName().equals(MaryXML.BOUNDARY)) { - segmentsAndBoundaries.add(tOrB); - } else { - assert tOrB.getTagName().equals(MaryXML.TOKEN) : "Expected token, got " + tOrB.getTagName(); - NodeList segs = tOrB.getElementsByTagName(MaryXML.PHONE); - for (int i = 0, max = segs.getLength(); i < max; i++) { - segmentsAndBoundaries.add((Element) segs.item(i)); - } - } - } - - List targets = createTargets(segmentsAndBoundaries); - // compute target features for each target in the chain - TargetCostFunction tcf = database.getTargetCostFunction(); - for (Target target : targets) { - tcf.computeTargetFeatures(target); - } - - Viterbi viterbi; - // Select the best candidates using Viterbi and the join cost function. - if (sCostWeight < 0) { - viterbi = new Viterbi(targets, database, targetCostWeight, beamSize); - } else { - viterbi = new Viterbi(targets, database, targetCostWeight, sCostWeight, beamSize); - } - - viterbi.apply(); - List selectedUnits = viterbi.getSelectedUnits(); - // If you can not associate the candidate units in the best path - // with the items in the segment relation, there is no best path - if (selectedUnits == null) { - throw new IllegalStateException("Viterbi: can't find path"); - } - long newtime = System.currentTimeMillis() - time; - logger.debug("Selection took " + newtime + " milliseconds"); - return selectedUnits; - } - - /** - * Create the list of targets from the XML elements to synthesize. - * - * @param segmentsAndBoundaries - * a list of MaryXML phone and boundary elements - * @return a list of Target objects - */ - protected List createTargets(List segmentsAndBoundaries) { - List targets = new ArrayList(); - for (Element sOrB : segmentsAndBoundaries) { - String phone = getPhoneSymbol(sOrB); - targets.add(new Target(phone, sOrB)); - } - return targets; - } - - public static String getPhoneSymbol(Element segmentOrBoundary) { - String phone; - if (segmentOrBoundary.getTagName().equals(MaryXML.PHONE)) { - phone = segmentOrBoundary.getAttribute("p"); - } else { - assert segmentOrBoundary.getTagName().equals(MaryXML.BOUNDARY) : "Expected boundary element, but got " - + segmentOrBoundary.getTagName(); - // TODO: how can we know the silence symbol here? - phone = "_"; - } - return phone; - } - -} diff --git a/marytts-runtime/src/main/java/marytts/unitselection/select/viterbi/Viterbi.java b/marytts-runtime/src/main/java/marytts/unitselection/select/viterbi/Viterbi.java deleted file mode 100644 index 9e63ef9637..0000000000 --- a/marytts-runtime/src/main/java/marytts/unitselection/select/viterbi/Viterbi.java +++ /dev/null @@ -1,532 +0,0 @@ -/** - * Portions Copyright 2006 DFKI GmbH. - * Portions Copyright 2001 Sun Microsystems, Inc. - * Portions Copyright 1999-2001 Language Technologies Institute, - * Carnegie Mellon University. - * All Rights Reserved. Use is subject to license terms. - * - * Permission is hereby granted, free of charge, to use and distribute - * this software and its documentation without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of this work, and to - * permit persons to whom this work is furnished to do so, subject to - * the following conditions: - * - * 1. The code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * 2. Any modifications must be clearly marked as such. - * 3. Original authors' names are not deleted. - * 4. The authors' names are not used to endorse or promote products - * derived from this software without specific prior written - * permission. - * - * DFKI GMBH AND THE CONTRIBUTORS TO THIS WORK DISCLAIM ALL WARRANTIES WITH - * REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL DFKI GMBH NOR THE - * CONTRIBUTORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL - * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR - * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS - * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF - * THIS SOFTWARE. - */ -package marytts.unitselection.select.viterbi; - -import java.io.PrintWriter; -import java.io.StringWriter; -import java.text.DecimalFormat; -import java.util.Collections; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; - -import marytts.exceptions.SynthesisException; -import marytts.unitselection.data.DiphoneUnit; -import marytts.unitselection.data.Unit; -import marytts.unitselection.data.UnitDatabase; -import marytts.unitselection.select.DiphoneTarget; -import marytts.unitselection.select.JoinCostFunction; -import marytts.unitselection.select.SelectedUnit; -import marytts.unitselection.select.StatisticalCostFunction; -import marytts.unitselection.select.Target; -import marytts.unitselection.select.TargetCostFunction; -import marytts.util.MaryUtils; - -import org.apache.log4j.Level; -import org.apache.log4j.Logger; - -/** - * Provides support for the Viterbi Algorithm. - * - * Implementation Notes - *

- * For each candidate for the current unit, calculate the cost between it and the first candidate in the next unit. Save only the - * path that has the least cost. By default, if two candidates come from units that are adjacent in the database, the cost is 0 - * (i.e., they were spoken together, so they are a perfect match). - *

- * - * Repeat the previous process for each candidate in the next unit, creating a list of least cost paths between the candidates - * between the current unit and the unit following it. - *

- * - * Toss out all candidates in the current unit that are not included in a path. - *

- * - * Move to the next unit and repeat the process. - */ -public class Viterbi { - // a general flag indicating which type of viterbi search - // to use: - // -1: unlimited search - // n>0: beam search, retain only the n best paths at each step. - protected int beamSize; - protected final float wTargetCosts; - protected final float wJoinCosts; - protected final float wSCosts; - - protected ViterbiPoint firstPoint = null; - protected ViterbiPoint lastPoint = null; - private UnitDatabase database; - protected TargetCostFunction targetCostFunction; - protected JoinCostFunction joinCostFunction; - protected StatisticalCostFunction sCostFunction; - protected Logger logger; - // for debugging, try to get an idea of the average effect of join vs. target costs: - protected double cumulJoinCosts; - protected int nJoinCosts; - protected double cumulTargetCosts; - protected int nTargetCosts; - - // Keep track of average costs for each voice: map UnitDatabase->DebugStats - private static Map debugStats = new HashMap(); - - /** - * Creates a Viterbi class to process the given utterance. A queue of ViterbiPoints corresponding to the Items in the Relation - * segs is built up. - * - */ - public Viterbi(List targets, UnitDatabase database, float wTargetCosts, int beamSize) { - this.database = database; - this.targetCostFunction = database.getTargetCostFunction(); - this.joinCostFunction = database.getJoinCostFunction(); - this.sCostFunction = database.getSCostFunction(); - this.logger = MaryUtils.getLogger("Viterbi"); - this.wTargetCosts = wTargetCosts; - wJoinCosts = 1 - wTargetCosts; - wSCosts = 0; - this.beamSize = beamSize; - this.cumulJoinCosts = 0; - this.nJoinCosts = 0; - this.cumulTargetCosts = 0; - this.nTargetCosts = 0; - ViterbiPoint last = null; - // for each segment, build a ViterbiPoint - for (Target target : targets) { - ViterbiPoint nextPoint = new ViterbiPoint(target); - - if (last != null) { // continue to build up the queue - last.setNext(nextPoint); - } else { // firstPoint is the start of the queue - firstPoint = nextPoint; - // dummy start path: - firstPoint.getPaths().add(new ViterbiPath(null, null, 0)); - } - last = nextPoint; - } - // And add one point where the paths from the last candidate can end: - lastPoint = new ViterbiPoint(null); - last.setNext(lastPoint); - if (beamSize == 0) { - throw new IllegalStateException("General beam search not implemented"); - } - } - - /** - * Creates a Viterbi class to process the given utterance. A queue of ViterbiPoints corresponding to the Items in the Relation - * segs is built up. - * - */ - public Viterbi(List targets, UnitDatabase database, float wTargetCosts, float wSCosts, int beamSize) { - this.database = database; - this.targetCostFunction = database.getTargetCostFunction(); - this.joinCostFunction = database.getJoinCostFunction(); - this.sCostFunction = database.getSCostFunction(); - this.logger = MaryUtils.getLogger("Viterbi"); - this.wTargetCosts = wTargetCosts; - this.wSCosts = wSCosts; - wJoinCosts = 1 - (wTargetCosts + wSCosts); - this.beamSize = beamSize; - this.cumulJoinCosts = 0; - this.nJoinCosts = 0; - this.cumulTargetCosts = 0; - this.nTargetCosts = 0; - ViterbiPoint last = null; - // for each segment, build a ViterbiPoint - for (Target target : targets) { - ViterbiPoint nextPoint = new ViterbiPoint(target); - - if (last != null) { // continue to build up the queue - last.setNext(nextPoint); - } else { // firstPoint is the start of the queue - firstPoint = nextPoint; - // dummy start path: - firstPoint.getPaths().add(new ViterbiPath(null, null, 0)); - } - last = nextPoint; - } - // And add one point where the paths from the last candidate can end: - lastPoint = new ViterbiPoint(null); - last.setNext(lastPoint); - if (beamSize == 0) { - throw new IllegalStateException("General beam search not implemented"); - } - } - - /** - * Carry out a Viterbi search in for a prepared queue of ViterbiPoints. In a nutshell, each Point represents a target item (a - * target segment); for each target Point, a number of Candidate units in the voice database are determined; a Path structure - * is built up, based on local best transitions. Concretely, a Path consists of a (possibly empty) previous Path, a current - * Candidate, and a Score. This Score is a quality measure of the Path; it is calculated as the sum of the previous Path's - * score, the Candidate's score, and the Cost of joining the Candidate to the previous Path's Candidate. At each step, only - * one Path leading to each Candidate is retained, viz. the Path with the best Score. All that is left to do is to call - * result() to get the best-rated path from among the paths associated with the last Point, and to associate the resulting - * Candidates with the segment items they will realise. - * - * @throws SynthesisException - * if for any part of the target chain, no candidates can be found - */ - public void apply() throws SynthesisException { - logger.debug("Viterbi running with beam size " + beamSize); - // go through all but the last point - // (since last point has no item) - for (ViterbiPoint point = firstPoint; point.next != null; point = point.next) { - // The candidates for the current item: - // candidate selection is carried out by UnitSelector - Target target = point.target; - List candidates = database.getCandidates(target); - if (candidates.size() == 0) { - if (target instanceof DiphoneTarget) { - logger.debug("No diphone '" + target.getName() + "' -- will build from halfphones"); - DiphoneTarget dt = (DiphoneTarget) target; - // replace diphone viterbi point with two half-phone viterbi points - Target left = dt.left; - Target right = dt.right; - point.setTarget(left); - ViterbiPoint newP = new ViterbiPoint(right); - newP.next = point.next; - point.next = newP; - candidates = database.getCandidates(left); - if (candidates.size() == 0) - throw new SynthesisException("Cannot even find any halfphone unit for target " + left); - } else { - throw new SynthesisException("Cannot find any units for target " + target); - } - } - assert candidates.size() > 0; - - // absolutely critical since candidates is no longer a SortedSet: - Collections.sort(candidates); - - point.candidates = candidates; - assert beamSize != 0; // general beam search not implemented - - // Now go through all existing paths and all candidates - // for the current item; - // tentatively extend each existing path to each of - // the candidates, but only retain the best one - List paths = point.paths; - int nPaths = paths.size(); - if (beamSize != -1 && beamSize < nPaths) { - // beam search, look only at the best n paths: - nPaths = beamSize; - } - // for searchStrategy == -1, no beam -- look at all candidates. - int i = 0; - int iMax = nPaths; - for (ViterbiPath pp : paths) { - assert pp != null; - // We are at the very beginning of the search, - // or have a usable path to extend - candidates = point.candidates; - assert candidates != null; - int j = 0; - int jMax = beamSize; - // Go through the candidates as returned by the iterator of the sorted set, - // i.e. sorted according to increasing target cost. - for (ViterbiCandidate c : candidates) { - // For the candidate c, create a path extending the - // previous path pp to that candidate, taking into - // account the target and join costs: - ViterbiPath np = getPath(pp, c); - // Compare this path to the existing best path - // (if any) leading to candidate c; only retain - // the one with the better score. - addPath(point.next, np); - if (++j == jMax) - break; - } - if (++i == iMax) - break; - } - } - } - - /** - * Add the new path to the state path if it is better than the current path. In this, state means the position of the - * candidate associated with this path in the candidate queue for the corresponding segment item. In other words, this method - * uses newPath as the one path leading to the candidate newPath.candidate, if it has a better score than the previously best - * path leading to that candidate. - * - * @param point - * where the path is added - * @param newPath - * the path to add if its score is best - */ - void addPath(ViterbiPoint point, ViterbiPath newPath) { - // get the position of newPath's candidate - // in path array statePath of point - ViterbiCandidate candidate = newPath.candidate; - assert candidate != null; - ViterbiPath bestPathSoFar = candidate.bestPath; - List paths = point.getPaths(); - if (bestPathSoFar == null) { - // we don't have a path for the candidate yet, so this is best - paths.add(newPath); - candidate.setBestPath(newPath); - } else if (newPath.score < bestPathSoFar.score) { - // newPath is a better path for the candidate - paths.remove(bestPathSoFar); - paths.add(newPath); - candidate.setBestPath(newPath); - } - } - - /** - * Collect and return the best path, as a List of SelectedUnit objects. Note: This is a replacement for result(). - * - * @return the list of selected units, or null if no path could be found. - */ - public List getSelectedUnits() { - LinkedList selectedUnits = new LinkedList(); - if (firstPoint == null || firstPoint.getNext() == null) { - return selectedUnits; // null case - } - ViterbiPath best = findBestPath(); - if (best == null) { - // System.out.println("No best path found"); - return null; - } - for (ViterbiPath path = best; path != null; path = path.getPrevious()) { - if (path.candidate != null) { - Unit u = path.candidate.unit; - Target t = path.candidate.target; - if (u instanceof DiphoneUnit) { - assert t instanceof DiphoneTarget; - DiphoneUnit du = (DiphoneUnit) u; - DiphoneTarget dt = (DiphoneTarget) t; - selectedUnits.addFirst(new SelectedUnit(du.right, dt.right)); - selectedUnits.addFirst(new SelectedUnit(du.left, dt.left)); - } else { - selectedUnits.addFirst(new SelectedUnit(u, t)); - } - } - } - if (logger.getEffectiveLevel().equals(Level.DEBUG)) { - StringWriter sw = new StringWriter(); - PrintWriter pw = new PrintWriter(sw); - int prevIndex = -1; // index number of the previous unit - int[] lengthHistogram = new int[10]; - int length = 0; - int numUnits = selectedUnits.size(); - StringBuilder line = new StringBuilder(); - for (int i = 0; i < numUnits; i++) { - SelectedUnit u = (SelectedUnit) selectedUnits.get(i); - int index = u.getUnit().index; - if (prevIndex + 1 == index) { // adjacent units - length++; - } else { - if (lengthHistogram.length <= length) { - int[] dummy = new int[length + 1]; - System.arraycopy(lengthHistogram, 0, dummy, 0, lengthHistogram.length); - lengthHistogram = dummy; - } - lengthHistogram[length]++; - pw.print(line); - // Find filename from which the stretch that just finished - // stems: - if (i > 0) { - assert i >= length; - Unit firstUnitInStretch = ((SelectedUnit) selectedUnits.get(i - length)).getUnit(); - String origin = database.getFilenameAndTime(firstUnitInStretch); - // Print origin from column 80: - for (int col = line.length(); col < 80; col++) - pw.print(" "); - pw.print(origin); - } - pw.println(); - length = 1; - line.setLength(0); - } - line.append(database.getTargetCostFunction().getFeature(u.getUnit(), "phone") + "(" + u.getUnit().index + ")"); - prevIndex = index; - } - if (lengthHistogram.length <= length) { - int[] dummy = new int[length + 1]; - System.arraycopy(lengthHistogram, 0, dummy, 0, lengthHistogram.length); - lengthHistogram = dummy; - } - lengthHistogram[length]++; - pw.print(line); - // Find filename from which the stretch that just finished - // stems: - Unit firstUnitInStretch = ((SelectedUnit) selectedUnits.get(numUnits - length)).getUnit(); - String origin = database.getFilenameAndTime(firstUnitInStretch); - // Print origin from column 80: - for (int col = line.length(); col < 80; col++) - pw.print(" "); - pw.print(origin); - pw.println(); - logger.debug("Selected units:\n" + sw.toString()); - // Compute average length of stretches: - int total = 0; - int nStretches = 0; - for (int l = 1; l < lengthHistogram.length; l++) { - // lengthHistogram[0] will be 0 anyway - total += lengthHistogram[l] * l; - nStretches += lengthHistogram[l]; - } - float avgLength = total / (float) nStretches; - DecimalFormat df = new DecimalFormat("0.000"); - logger.debug("Avg. consecutive length: " + df.format(avgLength) + " units"); - // Cost of best path - double totalCost = best.score; - int elements = selectedUnits.size(); - double avgCostBestPath = totalCost / (elements - 1); - double avgTargetCost = cumulTargetCosts / nTargetCosts; - double avgJoinCost = cumulJoinCosts / nJoinCosts; - logger.debug("Avg. cost: best path " + df.format(avgCostBestPath) + ", avg. target " + df.format(avgTargetCost) - + ", join " + df.format(avgJoinCost) + " (n=" + nTargetCosts + ")"); - DebugStats stats = debugStats.get(database); - if (stats == null) { - stats = new DebugStats(); - debugStats.put(database, stats); - } - stats.n++; - // iterative computation of mean: - // m(n) = m(n-1) + (x(n) - m(n-1)) / n - stats.avgLength += (avgLength - stats.avgLength) / stats.n; - stats.avgCostBestPath += (avgCostBestPath - stats.avgCostBestPath) / stats.n; - stats.avgTargetCost += (avgTargetCost - stats.avgTargetCost) / stats.n; - stats.avgJoinCost += (avgJoinCost - stats.avgJoinCost) / stats.n; - logger.debug("Total average of " + stats.n + " utterances for this voice:"); - logger.debug("Avg. length: " + df.format(stats.avgLength) + ", avg. cost best path: " - + df.format(stats.avgCostBestPath) + ", avg. target cost: " + df.format(stats.avgTargetCost) - + ", avg. join cost: " + df.format(stats.avgJoinCost)); - - } - - return selectedUnits; - } - - /** - * Construct a new path element linking a previous path to the given candidate. The (penalty) score associated with the new - * path is calculated as the sum of the score of the old path plus the score of the candidate itself plus the join cost of - * appending the candidate to the nearest candidate in the given path. This join cost takes into account optimal coupling if - * the database has OPTIMAL_COUPLING set to 1. The join position is saved in the new path, as the features "unit_prev_move" - * and "unit_this_move". - * - * @param path - * the previous path, or null if this candidate starts a new path - * @param candiate - * the candidate to add to the path - * - * @return a new path, consisting of this candidate appended to the previous path, and with the cumulative (penalty) score - * calculated. - */ - private ViterbiPath getPath(ViterbiPath path, ViterbiCandidate candidate) { - double cost; - - Target candidateTarget = candidate.target; - Unit candidateUnit = candidate.unit; - - double joinCost; - double sCost = 0; - double targetCost; - // Target costs: - targetCost = candidate.targetCost; - - if (path == null || path.candidate == null) { - joinCost = 0; - } else { - // Join costs: - ViterbiCandidate prevCandidate = path.candidate; - Target prevTarget = prevCandidate.target; - Unit prevUnit = prevCandidate.unit; - joinCost = joinCostFunction.cost(prevTarget, prevUnit, candidateTarget, candidateUnit); - if (sCostFunction != null) - sCost = sCostFunction.cost(prevUnit, candidateUnit); - } - // Total cost is a weighted sum of join cost and target cost: - // cost = (1-r) * joinCost + r * targetCost, - // where r is given as the property "viterbi.wTargetCost" in a config file. - targetCost *= wTargetCosts; - joinCost *= wJoinCosts; - sCost *= wSCosts; - cost = joinCost + targetCost + sCost; - if (joinCost < Float.POSITIVE_INFINITY) - cumulJoinCosts += joinCost; - nJoinCosts++; - cumulTargetCosts += targetCost; - nTargetCosts++; - // logger.debug(candidateUnit+": target cost "+targetCost+", join cost "+joinCost); - - if (path != null) { - cost += path.score; - } - - return new ViterbiPath(candidate, path, cost); - } - - /** - * Find the best path. This requires apply() to have been run. For this best path, we set the pointers to the *next* path - * elements correctly. - * - * @return the best path, or null if no best path could be found. - */ - private ViterbiPath findBestPath() { - assert beamSize != 0; - // All paths end in lastPoint, and take into account - // previous path segment's scores. Therefore, it is - // sufficient to find the best path from among the - // paths for lastPoint. - List paths = lastPoint.getPaths(); - if (paths.isEmpty()) // no path, we failed - return null; - - // as paths is no longer a SortedSet, they must be explicitly sorted: - Collections.sort(paths); - ViterbiPath best = paths.get(0); - - // Set *next* pointers correctly: - ViterbiPath path = best; - double totalCost = best.score; - int elements = 0; - while (path != null) { - elements++; - ViterbiPath prev = path.previous; - if (prev != null) - prev.setNext(path); - path = prev; - } - return best; - } - - private class DebugStats { - int n; - double avgLength; - double avgCostBestPath; - double avgTargetCost; - double avgJoinCost; - } - -} diff --git a/marytts-runtime/src/main/java/marytts/unitselection/select/viterbi/ViterbiCandidate.java b/marytts-runtime/src/main/java/marytts/unitselection/select/viterbi/ViterbiCandidate.java deleted file mode 100644 index 53206573c2..0000000000 --- a/marytts-runtime/src/main/java/marytts/unitselection/select/viterbi/ViterbiCandidate.java +++ /dev/null @@ -1,142 +0,0 @@ -/** - * Portions Copyright 2006 DFKI GmbH. - * Portions Copyright 2001 Sun Microsystems, Inc. - * Portions Copyright 1999-2001 Language Technologies Institute, - * Carnegie Mellon University. - * All Rights Reserved. Use is subject to license terms. - * - * Permission is hereby granted, free of charge, to use and distribute - * this software and its documentation without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of this work, and to - * permit persons to whom this work is furnished to do so, subject to - * the following conditions: - * - * 1. The code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * 2. Any modifications must be clearly marked as such. - * 3. Original authors' names are not deleted. - * 4. The authors' names are not used to endorse or promote products - * derived from this software without specific prior written - * permission. - * - * DFKI GMBH AND THE CONTRIBUTORS TO THIS WORK DISCLAIM ALL WARRANTIES WITH - * REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL DFKI GMBH NOR THE - * CONTRIBUTORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL - * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR - * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS - * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF - * THIS SOFTWARE. - */ -package marytts.unitselection.select.viterbi; - -import marytts.unitselection.data.Unit; -import marytts.unitselection.select.Target; -import marytts.unitselection.select.TargetCostFunction; - -/** - * Represents a candidate for the Viterbi algorthm. Each candidate knows about its next candidate, i.e. they can form a queue. - */ -public class ViterbiCandidate implements Comparable { - final Target target; - final Unit unit; - final double targetCost; - ViterbiPath bestPath = null; - ViterbiCandidate next = null; - - public ViterbiCandidate(Target target, Unit unit, TargetCostFunction tcf) { - this.target = target; - this.unit = unit; - this.targetCost = tcf.cost(target, unit); - } - - /** - * Calculates and returns the target cost for this candidate - * - * @param tcf - * the target cost function - * @return the target cost - */ - public double getTargetCost() { - return targetCost; - } - - /** - * Gets the next candidate in the queue - * - * @return the next candidate - */ - public ViterbiCandidate getNext() { - return next; - } - - /** - * Sets the next candidate in the queue - * - * @param next - * the next candidate - */ - public void setNext(ViterbiCandidate next) { - this.next = next; - } - - /** - * Gets the target of this candidate - * - * @return the target - */ - public Target getTarget() { - return target; - } - - /** - * Gets the index of this - * - * @return the unit index - */ - public Unit getUnit() { - return unit; - } - - /** - * Sets the currently best path leading to this candidate. Each path leads to exactly one candidate; in the candidate, we only - * remember the best path leading to it. - * - * @param bestPath - */ - public void setBestPath(ViterbiPath bestPath) { - this.bestPath = bestPath; - } - - /** - * Gets the best path leading to this candidate - * - * @return the best path, or null - */ - public ViterbiPath getBestPath() { - return bestPath; - } - - /** - * Converts this object to a string. - * - * @return the string form of this object - */ - public String toString() { - return "ViterbiCandidate: target " + target + ", unit " + unit - + (bestPath != null ? ", best path score " + bestPath.score : ", no best path"); - } - - /** - * Compare two candidates so that the one with the smaller target cost is considered smaller. - */ - public int compareTo(ViterbiCandidate o) { - if (targetCost < o.targetCost) { - return -1; - } else if (targetCost > o.targetCost) { - return 1; - } - return 0; - } -} diff --git a/marytts-runtime/src/main/java/marytts/unitselection/select/viterbi/ViterbiPath.java b/marytts-runtime/src/main/java/marytts/unitselection/select/viterbi/ViterbiPath.java deleted file mode 100644 index 32ebaa09ec..0000000000 --- a/marytts-runtime/src/main/java/marytts/unitselection/select/viterbi/ViterbiPath.java +++ /dev/null @@ -1,110 +0,0 @@ -/** - * Portions Copyright 2006 DFKI GmbH. - * Portions Copyright 2001 Sun Microsystems, Inc. - * Portions Copyright 1999-2001 Language Technologies Institute, - * Carnegie Mellon University. - * All Rights Reserved. Use is subject to license terms. - * - * Permission is hereby granted, free of charge, to use and distribute - * this software and its documentation without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of this work, and to - * permit persons to whom this work is furnished to do so, subject to - * the following conditions: - * - * 1. The code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * 2. Any modifications must be clearly marked as such. - * 3. Original authors' names are not deleted. - * 4. The authors' names are not used to endorse or promote products - * derived from this software without specific prior written - * permission. - * - * DFKI GMBH AND THE CONTRIBUTORS TO THIS WORK DISCLAIM ALL WARRANTIES WITH - * REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL DFKI GMBH NOR THE - * CONTRIBUTORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL - * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR - * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS - * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF - * THIS SOFTWARE. - */ -package marytts.unitselection.select.viterbi; - -/** - * Describes a Viterbi path. - */ -public class ViterbiPath implements Comparable { - final double score; - final ViterbiCandidate candidate; - final ViterbiPath previous; - ViterbiPath next = null; - - public ViterbiPath(ViterbiCandidate candidate, ViterbiPath previousPath, double score) { - this.candidate = candidate; - this.previous = previousPath; - this.score = score; - } - - /** - * Get the score of this path - * - * @return the score - */ - public double getScore() { - return score; - } - - /** - * Get the candidate of this path. Each path leads to exactly one candidate. - * - * @return the candidate - */ - public ViterbiCandidate getCandidate() { - return candidate; - } - - /** - * Get the next path - * - * @return the next path - */ - public ViterbiPath getNext() { - return next; - } - - /** - * Set the next path - * - * @param next - * the next path - */ - public void setNext(ViterbiPath next) { - this.next = next; - } - - /** - * Get the previous path - * - * @return the previous path - */ - public ViterbiPath getPrevious() { - return previous; - } - - /** - * Converts this object to a string. - * - * @return the string form of this object - */ - public String toString() { - return "ViterbiPath score " + score + " leads to candidate unit " + candidate.getUnit(); - } - - /** - * Compare two viterbi paths such that the one with the lower score is considered smaller. - */ - public int compareTo(ViterbiPath o) { - return Double.compare(score, o.score); - } -} diff --git a/marytts-runtime/src/main/java/marytts/unitselection/select/viterbi/ViterbiPoint.java b/marytts-runtime/src/main/java/marytts/unitselection/select/viterbi/ViterbiPoint.java deleted file mode 100644 index a109eff9bb..0000000000 --- a/marytts-runtime/src/main/java/marytts/unitselection/select/viterbi/ViterbiPoint.java +++ /dev/null @@ -1,129 +0,0 @@ -/** - * Portions Copyright 2006 DFKI GmbH. - * Portions Copyright 2001 Sun Microsystems, Inc. - * Portions Copyright 1999-2001 Language Technologies Institute, - * Carnegie Mellon University. - * All Rights Reserved. Use is subject to license terms. - * - * Permission is hereby granted, free of charge, to use and distribute - * this software and its documentation without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of this work, and to - * permit persons to whom this work is furnished to do so, subject to - * the following conditions: - * - * 1. The code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * 2. Any modifications must be clearly marked as such. - * 3. Original authors' names are not deleted. - * 4. The authors' names are not used to endorse or promote products - * derived from this software without specific prior written - * permission. - * - * DFKI GMBH AND THE CONTRIBUTORS TO THIS WORK DISCLAIM ALL WARRANTIES WITH - * REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL DFKI GMBH NOR THE - * CONTRIBUTORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL - * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR - * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS - * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF - * THIS SOFTWARE. - */ -package marytts.unitselection.select.viterbi; - -import java.util.ArrayList; -import java.util.List; - -import marytts.unitselection.select.Target; - -/** - * Represents a point in the Viterbi path. A point corresponds to an item, e.g. a Segment. Each ViterbiPoint knows about its next - * ViterbiPoint, i.e. they can form a queue. - */ -public class ViterbiPoint { - Target target = null; - List candidates = null; - List paths = new ArrayList(); - ViterbiPoint next = null; - - /** - * Creates a ViterbiPoint for the given target. - * - * @param target - * the target of interest - */ - public ViterbiPoint(Target target) { - this.target = target; - } - - /** - * Gets the target of this point - * - * @return the target - */ - public Target getTarget() { - return target; - } - - /** - * Sets the target of this point - * - * @param target - * the new target - */ - public void setTarget(Target target) { - this.target = target; - } - - /** - * Gets the candidates of this point - * - * @return the candidates - */ - public List getCandidates() { - return candidates; - } - - /** - * Sets the candidates of this point - * - * @param cands - * the candidates - */ - public void setCandidates(ArrayList candidates) { - this.candidates = candidates; - } - - /** - * Gets the sorted set containting the paths of the candidates of this point, sorted by score. getPaths().first() will return - * the path with the lowest score, i.e. the best path. - * - * @return a sorted set. - */ - public List getPaths() { - return paths; - } - - /** - * Gets the next point in the queue - * - * @return the next point - */ - public ViterbiPoint getNext() { - return next; - } - - /** - * Sets the next point in the queue - * - * @param next - * the next point - */ - public void setNext(ViterbiPoint next) { - this.next = next; - } - - public String toString() { - return "ViterbiPoint: target " + target + "; " + paths.size() + " paths"; - } -} diff --git a/marytts-runtime/src/main/java/marytts/unitselection/weightingfunctions/WeightFunc.java b/marytts-runtime/src/main/java/marytts/unitselection/weightingfunctions/WeightFunc.java deleted file mode 100644 index 991027d64a..0000000000 --- a/marytts-runtime/src/main/java/marytts/unitselection/weightingfunctions/WeightFunc.java +++ /dev/null @@ -1,49 +0,0 @@ -/** - * Portions Copyright 2006 DFKI GmbH. - * Portions Copyright 2001 Sun Microsystems, Inc. - * Portions Copyright 1999-2001 Language Technologies Institute, - * Carnegie Mellon University. - * All Rights Reserved. Use is subject to license terms. - * - * Permission is hereby granted, free of charge, to use and distribute - * this software and its documentation without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of this work, and to - * permit persons to whom this work is furnished to do so, subject to - * the following conditions: - * - * 1. The code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * 2. Any modifications must be clearly marked as such. - * 3. Original authors' names are not deleted. - * 4. The authors' names are not used to endorse or promote products - * derived from this software without specific prior written - * permission. - * - * DFKI GMBH AND THE CONTRIBUTORS TO THIS WORK DISCLAIM ALL WARRANTIES WITH - * REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL DFKI GMBH NOR THE - * CONTRIBUTORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL - * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR - * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS - * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF - * THIS SOFTWARE. - */ -package marytts.unitselection.weightingfunctions; - -/** - * A uniform interface for the weighting functions. - * - * @author sacha - * - */ -public interface WeightFunc { - /** Compute a weighted difference */ - public double cost(double a, double b); - - /** Possibly set optional parameters. */ - public void setParam(String val); - - /** Return a weighting function definition string. */ - public String whoAmI(); -} diff --git a/marytts-runtime/src/main/java/marytts/unitselection/weightingfunctions/WeightFunctionManager.java b/marytts-runtime/src/main/java/marytts/unitselection/weightingfunctions/WeightFunctionManager.java deleted file mode 100644 index ad7c767d9e..0000000000 --- a/marytts-runtime/src/main/java/marytts/unitselection/weightingfunctions/WeightFunctionManager.java +++ /dev/null @@ -1,99 +0,0 @@ -/** - * Portions Copyright 2006 DFKI GmbH. - * Portions Copyright 2001 Sun Microsystems, Inc. - * Portions Copyright 1999-2001 Language Technologies Institute, - * Carnegie Mellon University. - * All Rights Reserved. Use is subject to license terms. - * - * Permission is hereby granted, free of charge, to use and distribute - * this software and its documentation without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of this work, and to - * permit persons to whom this work is furnished to do so, subject to - * the following conditions: - * - * 1. The code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * 2. Any modifications must be clearly marked as such. - * 3. Original authors' names are not deleted. - * 4. The authors' names are not used to endorse or promote products - * derived from this software without specific prior written - * permission. - * - * DFKI GMBH AND THE CONTRIBUTORS TO THIS WORK DISCLAIM ALL WARRANTIES WITH - * REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL DFKI GMBH NOR THE - * CONTRIBUTORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL - * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR - * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS - * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF - * THIS SOFTWARE. - */ - -package marytts.unitselection.weightingfunctions; - -import java.util.HashMap; -import java.util.Map; - -/** - * This class connects weighting function names with the actual instances of the weighting functions. - * - * @author sacha - * - */ -public class WeightFunctionManager { - - public static Map weightFuncMap; - - static { - weightFuncMap = new HashMap(); - weightFuncMap.put("linear", new WeightingFunction.linear()); - weightFuncMap.put("step", new WeightingFunction.step()); - } - - /** - * Dummy empty contructor. - */ - public WeightFunctionManager() { - } - - /** - * Accessor for the hash map mapping names to interface instances. - * - * @return a hash map. - */ - public static Map getWeightFunc() { - return weightFuncMap; - } - - /** - * Returns the weighting function from its name. - * - * @param funcName - * The name of the weighting function. - * @return an interface to a weighting function. - */ - public WeightFunc getWeightFunction(String funcName) { - /* Split the string in 2 parts: function name plus parameters */ - String[] strPart = funcName.split("\\s", 2); - WeightFunc wf = (WeightFunc) weightFuncMap.get(strPart[0]); - /* If the function asked for does not exist, inform the user and throw an exception. */ - if (wf == null) { - String[] known = (String[]) weightFuncMap.keySet().toArray(new String[0]); - String strKnown = known[0]; - for (int i = 1; i < known.length; i++) { - strKnown = strKnown + "; " + known[i]; - } - strKnown = strKnown + "."; - throw new RuntimeException("The weighting manager was asked for the unknown weighting function type [" + funcName - + "]. Known types are: " + strKnown); - } - /* If the function has a parameter, parse and set it */ - // TODO: This is not thread-safe! What if several threads call wf.setParam() with different values? - if (strPart.length > 1) - wf.setParam(strPart[1]); - /* Return the functionś interface */ - return (wf); - } - -} diff --git a/marytts-runtime/src/main/java/marytts/unitselection/weightingfunctions/WeightingFunction.java b/marytts-runtime/src/main/java/marytts/unitselection/weightingfunctions/WeightingFunction.java deleted file mode 100644 index d0b75996c1..0000000000 --- a/marytts-runtime/src/main/java/marytts/unitselection/weightingfunctions/WeightingFunction.java +++ /dev/null @@ -1,119 +0,0 @@ -/** - * Portions Copyright 2006 DFKI GmbH. - * Portions Copyright 2001 Sun Microsystems, Inc. - * Portions Copyright 1999-2001 Language Technologies Institute, - * Carnegie Mellon University. - * All Rights Reserved. Use is subject to license terms. - * - * Permission is hereby granted, free of charge, to use and distribute - * this software and its documentation without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of this work, and to - * permit persons to whom this work is furnished to do so, subject to - * the following conditions: - * - * 1. The code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * 2. Any modifications must be clearly marked as such. - * 3. Original authors' names are not deleted. - * 4. The authors' names are not used to endorse or promote products - * derived from this software without specific prior written - * permission. - * - * DFKI GMBH AND THE CONTRIBUTORS TO THIS WORK DISCLAIM ALL WARRANTIES WITH - * REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL DFKI GMBH NOR THE - * CONTRIBUTORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL - * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR - * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS - * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF - * THIS SOFTWARE. - */ -package marytts.unitselection.weightingfunctions; - -/** - * Defines the applicable weighting functions. - * - * @author sacha - * - */ -public class WeightingFunction { - - /** - * Linear weighting function: just computes the difference. - * - * @author sacha - * - */ - public static class linear implements WeightFunc { - /** - * Cost computation: a simple absolute value of a subtraction. - */ - public double cost(double a, double b) { - return (a > b ? (a - b) : (b - a)); - /* - * (Measuring which one is bigger replaces a costly Math.abs() operation.) - */ - } - - /** Optional argument setting. */ - public void setParam(String val) { - /* - * Does nothing and is never called in the linear case. - */ - } - - /** - * Returns the string definition for this weight function. - */ - public String whoAmI() { - return ("linear"); - } - } - - /** - * Step weighting function: saturates above a given percentage of the input values. - * - * @author sacha - * - */ - public static class step implements WeightFunc { - /** A percentage above which the function saturates to 0. */ - private double stepVal; - - /** - * Cost computation: the absolute value of a subtraction, with application of a saturation if the difference value reaches - * a certain percentage of the mean value of the arguments. - */ - public double cost(double a, double b) { - - double res = (a > b ? (a - b) : (b - a)); - /* - * (Measuring which one is bigger replaces a costly Math.abs() operation.) - */ - - double dev = res / ((a + b) / 2.0); - if (dev < stepVal) - return (res); - else - return (0.0); - } - - /** - * Optional argument setting. - * - * Syntax for the step function's parameter: the first number before the % sign is interpreted as a percentage for the - * step value. - * - * */ - public void setParam(String val) { - stepVal = Double.parseDouble(val.substring(0, val.indexOf("%"))) / 100.0; - } - - /** String definition of the function. */ - public String whoAmI() { - return ("step " + Double.toString(100.0 * stepVal) + "%"); - } - } - -} From a2a9b5a87efaaa27acae82b07ab035a1b02418dc Mon Sep 17 00:00:00 2001 From: Tristan Hamilton Date: Thu, 5 Feb 2015 14:25:49 +0100 Subject: [PATCH 06/31] moved unitselection into new maven module marytts-unitselection. copied cart and features packages into new maven module marytts-unitselection (but also still in runtime) moved Target, HalfPhoneTarget,DiphoneTarget and TargetFeatureComputer from unitselection.select to features package moved feature file readers from unitselection.data to features still has compile errors --- .../java/marytts/features/DiphoneTarget.java | 71 + .../marytts/features/FeatureFileReader.java | 207 ++ .../features/HalfPhoneFeatureFileReader.java | 99 + .../marytts/features/HalfPhoneTarget.java | 60 + .../main/java/marytts/features/Target.java | 191 ++ .../src/main/java/marytts/cart/CART.java | 177 + .../main/java/marytts/cart/DecisionNode.java | 748 ++++ .../main/java/marytts/cart/DirectedGraph.java | 276 ++ .../java/marytts/cart/DirectedGraphNode.java | 218 ++ .../java/marytts/cart/FeatureVectorCART.java | 129 + .../src/main/java/marytts/cart/LeafNode.java | 541 +++ .../src/main/java/marytts/cart/Node.java | 172 + .../main/java/marytts/cart/NodeIterator.java | 163 + .../marytts/cart/StringPredictionTree.java | 222 ++ .../cart/impose/FeatureArrayIndexer.java | 422 +++ .../cart/impose/FeatureComparator.java | 118 + .../impose/FeatureFileIndexingResult.java | 46 + .../java/marytts/cart/impose/MaryNode.java | 133 + .../marytts/cart/io/DirectedGraphReader.java | 290 ++ .../marytts/cart/io/DirectedGraphWriter.java | 409 +++ .../java/marytts/cart/io/HTSCARTReader.java | 556 +++ .../java/marytts/cart/io/MaryCARTReader.java | 402 +++ .../java/marytts/cart/io/MaryCARTWriter.java | 345 ++ .../java/marytts/cart/io/WagonCARTReader.java | 607 ++++ .../java/marytts/cart/io/WagonCARTWriter.java | 379 ++ .../features/ByteValuedFeatureProcessor.java | 48 + .../features/ContinuousFeatureProcessor.java | 41 + .../java/marytts/features/DiphoneTarget.java | 71 + .../marytts/features/FeatureDefinition.java | 1730 ++++++++++ .../features/FeatureProcessorManager.java | 344 ++ .../marytts/features/FeatureRegistry.java | 241 ++ .../java/marytts/features/FeatureVector.java | 326 ++ .../marytts/features/HalfPhoneTarget.java | 60 + .../features/MaryFeatureProcessor.java | 40 + .../MaryGenericFeatureProcessors.java | 3055 +++++++++++++++++ .../MaryLanguageFeatureProcessors.java | 468 +++ .../features/ShortValuedFeatureProcessor.java | 48 + .../main/java/marytts/features/Target.java | 191 ++ .../features/TargetFeatureComputer.java | 348 ++ .../UnitSelectionSynthesizer.java | 270 ++ .../unitselection/UnitSelectionVoice.java | 325 ++ .../analysis/HnmVoiceDataDumper.java | 126 + .../marytts/unitselection/analysis/Phone.java | 748 ++++ .../analysis/ProsodyAnalyzer.java | 437 +++ .../analysis/VoiceDataDumper.java | 351 ++ .../concat/BaseUnitConcatenator.java | 315 ++ .../DatagramOverlapDoubleDataSource.java | 148 + .../concat/FdpsolaUnitConcatenator.java | 601 ++++ .../concat/HnmUnitConcatenator.java | 251 ++ .../concat/OverlapUnitConcatenator.java | 206 ++ .../concat/UnitConcatenator.java | 61 + .../unitselection/data/DiphoneUnit.java | 65 + .../data/DiphoneUnitDatabase.java | 159 + .../unitselection/data/FeatureFileReader.java | 218 ++ .../data/FloatArrayDatagram.java | 67 + .../data/HalfPhoneFeatureFileReader.java | 99 + .../unitselection/data/HnmDatagram.java | 149 + .../unitselection/data/HnmTimelineReader.java | 248 ++ .../unitselection/data/LPCDatagram.java | 229 ++ .../unitselection/data/LPCTimelineReader.java | 100 + .../unitselection/data/MCepDatagram.java | 175 + .../data/MCepTimelineReader.java | 85 + .../unitselection/data/SCostFileReader.java | 140 + .../marytts/unitselection/data/Sentence.java | 63 + .../unitselection/data/SentenceIterator.java | 152 + .../marytts/unitselection/data/Syllable.java | 77 + .../unitselection/data/SyllableIterator.java | 151 + .../unitselection/data/TimelineReader.java | 1304 +++++++ .../java/marytts/unitselection/data/Unit.java | 85 + .../unitselection/data/UnitDatabase.java | 195 ++ .../unitselection/data/UnitFileReader.java | 200 ++ .../InterpolatingSynthesizer.java | 203 ++ .../interpolation/InterpolatingVoice.java | 119 + .../select/DiphoneFFRTargetCostFunction.java | 145 + .../select/DiphoneUnitSelector.java | 72 + .../select/FFRTargetCostFunction.java | 387 +++ .../HalfPhoneFFRTargetCostFunction.java | 245 ++ .../select/HalfPhoneUnitSelector.java | 59 + .../select/JoinCostFeatures.java | 626 ++++ .../select/JoinCostFunction.java | 78 + .../unitselection/select/JoinModelCost.java | 220 ++ .../select/PrecompiledJoinCostReader.java | 159 + .../unitselection/select/SelectedUnit.java | 77 + .../select/StatisticalCostFunction.java | 44 + .../select/StatisticalModelCost.java | 143 + .../select/TargetCostFunction.java | 127 + .../unitselection/select/UnitSelector.java | 159 + .../unitselection/select/viterbi/Viterbi.java | 532 +++ .../select/viterbi/ViterbiCandidate.java | 142 + .../select/viterbi/ViterbiPath.java | 110 + .../select/viterbi/ViterbiPoint.java | 129 + .../weightingfunctions/WeightFunc.java | 49 + .../WeightFunctionManager.java | 99 + .../weightingfunctions/WeightingFunction.java | 119 + 94 files changed, 25835 insertions(+) create mode 100644 marytts-runtime/src/main/java/marytts/features/DiphoneTarget.java create mode 100644 marytts-runtime/src/main/java/marytts/features/FeatureFileReader.java create mode 100644 marytts-runtime/src/main/java/marytts/features/HalfPhoneFeatureFileReader.java create mode 100644 marytts-runtime/src/main/java/marytts/features/HalfPhoneTarget.java create mode 100644 marytts-runtime/src/main/java/marytts/features/Target.java create mode 100644 marytts-unitselection/src/main/java/marytts/cart/CART.java create mode 100644 marytts-unitselection/src/main/java/marytts/cart/DecisionNode.java create mode 100644 marytts-unitselection/src/main/java/marytts/cart/DirectedGraph.java create mode 100644 marytts-unitselection/src/main/java/marytts/cart/DirectedGraphNode.java create mode 100644 marytts-unitselection/src/main/java/marytts/cart/FeatureVectorCART.java create mode 100644 marytts-unitselection/src/main/java/marytts/cart/LeafNode.java create mode 100644 marytts-unitselection/src/main/java/marytts/cart/Node.java create mode 100644 marytts-unitselection/src/main/java/marytts/cart/NodeIterator.java create mode 100644 marytts-unitselection/src/main/java/marytts/cart/StringPredictionTree.java create mode 100644 marytts-unitselection/src/main/java/marytts/cart/impose/FeatureArrayIndexer.java create mode 100644 marytts-unitselection/src/main/java/marytts/cart/impose/FeatureComparator.java create mode 100644 marytts-unitselection/src/main/java/marytts/cart/impose/FeatureFileIndexingResult.java create mode 100644 marytts-unitselection/src/main/java/marytts/cart/impose/MaryNode.java create mode 100644 marytts-unitselection/src/main/java/marytts/cart/io/DirectedGraphReader.java create mode 100644 marytts-unitselection/src/main/java/marytts/cart/io/DirectedGraphWriter.java create mode 100644 marytts-unitselection/src/main/java/marytts/cart/io/HTSCARTReader.java create mode 100644 marytts-unitselection/src/main/java/marytts/cart/io/MaryCARTReader.java create mode 100644 marytts-unitselection/src/main/java/marytts/cart/io/MaryCARTWriter.java create mode 100644 marytts-unitselection/src/main/java/marytts/cart/io/WagonCARTReader.java create mode 100644 marytts-unitselection/src/main/java/marytts/cart/io/WagonCARTWriter.java create mode 100644 marytts-unitselection/src/main/java/marytts/features/ByteValuedFeatureProcessor.java create mode 100644 marytts-unitselection/src/main/java/marytts/features/ContinuousFeatureProcessor.java create mode 100644 marytts-unitselection/src/main/java/marytts/features/DiphoneTarget.java create mode 100644 marytts-unitselection/src/main/java/marytts/features/FeatureDefinition.java create mode 100644 marytts-unitselection/src/main/java/marytts/features/FeatureProcessorManager.java create mode 100644 marytts-unitselection/src/main/java/marytts/features/FeatureRegistry.java create mode 100644 marytts-unitselection/src/main/java/marytts/features/FeatureVector.java create mode 100644 marytts-unitselection/src/main/java/marytts/features/HalfPhoneTarget.java create mode 100644 marytts-unitselection/src/main/java/marytts/features/MaryFeatureProcessor.java create mode 100644 marytts-unitselection/src/main/java/marytts/features/MaryGenericFeatureProcessors.java create mode 100644 marytts-unitselection/src/main/java/marytts/features/MaryLanguageFeatureProcessors.java create mode 100644 marytts-unitselection/src/main/java/marytts/features/ShortValuedFeatureProcessor.java create mode 100644 marytts-unitselection/src/main/java/marytts/features/Target.java create mode 100644 marytts-unitselection/src/main/java/marytts/features/TargetFeatureComputer.java create mode 100644 marytts-unitselection/src/main/java/marytts/unitselection/UnitSelectionSynthesizer.java create mode 100644 marytts-unitselection/src/main/java/marytts/unitselection/UnitSelectionVoice.java create mode 100644 marytts-unitselection/src/main/java/marytts/unitselection/analysis/HnmVoiceDataDumper.java create mode 100644 marytts-unitselection/src/main/java/marytts/unitselection/analysis/Phone.java create mode 100644 marytts-unitselection/src/main/java/marytts/unitselection/analysis/ProsodyAnalyzer.java create mode 100644 marytts-unitselection/src/main/java/marytts/unitselection/analysis/VoiceDataDumper.java create mode 100644 marytts-unitselection/src/main/java/marytts/unitselection/concat/BaseUnitConcatenator.java create mode 100644 marytts-unitselection/src/main/java/marytts/unitselection/concat/DatagramOverlapDoubleDataSource.java create mode 100644 marytts-unitselection/src/main/java/marytts/unitselection/concat/FdpsolaUnitConcatenator.java create mode 100644 marytts-unitselection/src/main/java/marytts/unitselection/concat/HnmUnitConcatenator.java create mode 100644 marytts-unitselection/src/main/java/marytts/unitselection/concat/OverlapUnitConcatenator.java create mode 100644 marytts-unitselection/src/main/java/marytts/unitselection/concat/UnitConcatenator.java create mode 100644 marytts-unitselection/src/main/java/marytts/unitselection/data/DiphoneUnit.java create mode 100644 marytts-unitselection/src/main/java/marytts/unitselection/data/DiphoneUnitDatabase.java create mode 100644 marytts-unitselection/src/main/java/marytts/unitselection/data/FeatureFileReader.java create mode 100644 marytts-unitselection/src/main/java/marytts/unitselection/data/FloatArrayDatagram.java create mode 100644 marytts-unitselection/src/main/java/marytts/unitselection/data/HalfPhoneFeatureFileReader.java create mode 100644 marytts-unitselection/src/main/java/marytts/unitselection/data/HnmDatagram.java create mode 100644 marytts-unitselection/src/main/java/marytts/unitselection/data/HnmTimelineReader.java create mode 100644 marytts-unitselection/src/main/java/marytts/unitselection/data/LPCDatagram.java create mode 100644 marytts-unitselection/src/main/java/marytts/unitselection/data/LPCTimelineReader.java create mode 100644 marytts-unitselection/src/main/java/marytts/unitselection/data/MCepDatagram.java create mode 100644 marytts-unitselection/src/main/java/marytts/unitselection/data/MCepTimelineReader.java create mode 100644 marytts-unitselection/src/main/java/marytts/unitselection/data/SCostFileReader.java create mode 100644 marytts-unitselection/src/main/java/marytts/unitselection/data/Sentence.java create mode 100644 marytts-unitselection/src/main/java/marytts/unitselection/data/SentenceIterator.java create mode 100644 marytts-unitselection/src/main/java/marytts/unitselection/data/Syllable.java create mode 100644 marytts-unitselection/src/main/java/marytts/unitselection/data/SyllableIterator.java create mode 100644 marytts-unitselection/src/main/java/marytts/unitselection/data/TimelineReader.java create mode 100644 marytts-unitselection/src/main/java/marytts/unitselection/data/Unit.java create mode 100644 marytts-unitselection/src/main/java/marytts/unitselection/data/UnitDatabase.java create mode 100644 marytts-unitselection/src/main/java/marytts/unitselection/data/UnitFileReader.java create mode 100644 marytts-unitselection/src/main/java/marytts/unitselection/interpolation/InterpolatingSynthesizer.java create mode 100644 marytts-unitselection/src/main/java/marytts/unitselection/interpolation/InterpolatingVoice.java create mode 100644 marytts-unitselection/src/main/java/marytts/unitselection/select/DiphoneFFRTargetCostFunction.java create mode 100644 marytts-unitselection/src/main/java/marytts/unitselection/select/DiphoneUnitSelector.java create mode 100644 marytts-unitselection/src/main/java/marytts/unitselection/select/FFRTargetCostFunction.java create mode 100644 marytts-unitselection/src/main/java/marytts/unitselection/select/HalfPhoneFFRTargetCostFunction.java create mode 100644 marytts-unitselection/src/main/java/marytts/unitselection/select/HalfPhoneUnitSelector.java create mode 100644 marytts-unitselection/src/main/java/marytts/unitselection/select/JoinCostFeatures.java create mode 100644 marytts-unitselection/src/main/java/marytts/unitselection/select/JoinCostFunction.java create mode 100644 marytts-unitselection/src/main/java/marytts/unitselection/select/JoinModelCost.java create mode 100644 marytts-unitselection/src/main/java/marytts/unitselection/select/PrecompiledJoinCostReader.java create mode 100644 marytts-unitselection/src/main/java/marytts/unitselection/select/SelectedUnit.java create mode 100644 marytts-unitselection/src/main/java/marytts/unitselection/select/StatisticalCostFunction.java create mode 100644 marytts-unitselection/src/main/java/marytts/unitselection/select/StatisticalModelCost.java create mode 100644 marytts-unitselection/src/main/java/marytts/unitselection/select/TargetCostFunction.java create mode 100644 marytts-unitselection/src/main/java/marytts/unitselection/select/UnitSelector.java create mode 100644 marytts-unitselection/src/main/java/marytts/unitselection/select/viterbi/Viterbi.java create mode 100644 marytts-unitselection/src/main/java/marytts/unitselection/select/viterbi/ViterbiCandidate.java create mode 100644 marytts-unitselection/src/main/java/marytts/unitselection/select/viterbi/ViterbiPath.java create mode 100644 marytts-unitselection/src/main/java/marytts/unitselection/select/viterbi/ViterbiPoint.java create mode 100644 marytts-unitselection/src/main/java/marytts/unitselection/weightingfunctions/WeightFunc.java create mode 100644 marytts-unitselection/src/main/java/marytts/unitselection/weightingfunctions/WeightFunctionManager.java create mode 100644 marytts-unitselection/src/main/java/marytts/unitselection/weightingfunctions/WeightingFunction.java diff --git a/marytts-runtime/src/main/java/marytts/features/DiphoneTarget.java b/marytts-runtime/src/main/java/marytts/features/DiphoneTarget.java new file mode 100644 index 0000000000..a97566e37f --- /dev/null +++ b/marytts-runtime/src/main/java/marytts/features/DiphoneTarget.java @@ -0,0 +1,71 @@ +/** + * Copyright 2006 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.features; + +import marytts.features.FeatureVector; +import marytts.modules.phonemiser.Allophone; + +import org.w3c.dom.Element; + +public class DiphoneTarget extends Target { + public final HalfPhoneTarget left; + public final HalfPhoneTarget right; + + public DiphoneTarget(HalfPhoneTarget left, HalfPhoneTarget right) { + super(null, null); + this.name = left.name.substring(0, left.name.lastIndexOf("_")) + "-" + + right.name.substring(0, right.name.lastIndexOf("_")); + assert left.isRightHalf(); // the left half of this diphone must be the right half of a phone + assert right.isLeftHalf(); + this.left = left; + this.right = right; + } + + @Override + public Element getMaryxmlElement() { + throw new IllegalStateException("This method should not be called for DiphoneTargets."); + } + + public FeatureVector getFeatureVector() { + throw new IllegalStateException("This method should not be called for DiphoneTargets."); + } + + public void setFeatureVector(FeatureVector featureVector) { + throw new IllegalStateException("This method should not be called for DiphoneTargets."); + } + + public float getTargetDurationInSeconds() { + throw new IllegalStateException("This method should not be called for DiphoneTargets."); + } + + /** + * Determine whether this target is a silence target + * + * @return true if the target represents silence, false otherwise + */ + public boolean isSilence() { + throw new IllegalStateException("This method should not be called for DiphoneTargets."); + } + + public Allophone getAllophone() { + throw new IllegalStateException("This method should not be called for DiphoneTargets."); + } + +} diff --git a/marytts-runtime/src/main/java/marytts/features/FeatureFileReader.java b/marytts-runtime/src/main/java/marytts/features/FeatureFileReader.java new file mode 100644 index 0000000000..2b338f9a8c --- /dev/null +++ b/marytts-runtime/src/main/java/marytts/features/FeatureFileReader.java @@ -0,0 +1,207 @@ +/** + * Copyright 2006 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.features; + +import java.io.BufferedInputStream; +import java.io.DataInputStream; +import java.io.FileInputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; + +import marytts.exceptions.MaryConfigurationException; +import marytts.features.FeatureDefinition; +import marytts.features.FeatureVector; +import marytts.util.data.MaryHeader; + +public class FeatureFileReader { + protected MaryHeader hdr; + protected FeatureDefinition featureDefinition; + protected FeatureVector[] featureVectors; + + /** + * Get a feature file reader representing the given feature file. + * + * @param fileName + * the filename of a valid feature file. + * @return a feature file object representing the given file. + * @throws IOException + * if there was a problem reading the file + * @throws MaryConfigurationException + * if the file is not a valid feature file. + */ + public static FeatureFileReader getFeatureFileReader(String fileName) throws IOException, MaryConfigurationException { + int fileType = MaryHeader.peekFileType(fileName); + if (fileType == MaryHeader.UNITFEATS) + return new FeatureFileReader(fileName); + else if (fileType == MaryHeader.HALFPHONE_UNITFEATS) + return new HalfPhoneFeatureFileReader(fileName); + throw new MaryConfigurationException("File " + fileName + ": Type " + fileType + " is not a known unit feature file type"); + } + + /** + * Empty constructor; need to call load() separately when using this. + * + * @see load(String) + */ + public FeatureFileReader() { + } + + public FeatureFileReader(String fileName) throws IOException, MaryConfigurationException { + load(fileName); + } + + public void load(String fileName) throws IOException, MaryConfigurationException { + loadFromByteBuffer(fileName); + } + + protected void loadFromStream(String fileName) throws IOException, MaryConfigurationException { + /* Open the file */ + DataInputStream dis = null; + dis = new DataInputStream(new BufferedInputStream(new FileInputStream(fileName))); + + /* Load the Mary header */ + hdr = new MaryHeader(dis); + if (hdr.getType() != MaryHeader.UNITFEATS && hdr.getType() != MaryHeader.HALFPHONE_UNITFEATS) { + throw new IOException("File [" + fileName + "] is not a valid Mary feature file."); + } + featureDefinition = new FeatureDefinition(dis); + int numberOfUnits = dis.readInt(); + featureVectors = new FeatureVector[numberOfUnits]; + for (int i = 0; i < numberOfUnits; i++) { + featureVectors[i] = featureDefinition.readFeatureVector(i, dis); + } + + } + + protected void loadFromByteBuffer(String fileName) throws IOException, MaryConfigurationException { + /* Open the file */ + FileInputStream fis = new FileInputStream(fileName); + FileChannel fc = fis.getChannel(); + ByteBuffer bb = fc.map(FileChannel.MapMode.READ_ONLY, 0, fc.size()); + fis.close(); + + /* Load the Mary header */ + hdr = new MaryHeader(bb); + if (hdr.getType() != MaryHeader.UNITFEATS && hdr.getType() != MaryHeader.HALFPHONE_UNITFEATS) { + throw new MaryConfigurationException("File [" + fileName + "] is not a valid Mary feature file."); + } + featureDefinition = new FeatureDefinition(bb); + int numberOfUnits = bb.getInt(); + featureVectors = new FeatureVector[numberOfUnits]; + for (int i = 0; i < numberOfUnits; i++) { + featureVectors[i] = featureDefinition.readFeatureVector(i, bb); + } + + } + + /** + * Get the unit feature vector for the given unit index number. + * + * @param unitIndex + * the absolute index number of a unit in the database + * @return the corresponding feature vector + */ + public FeatureVector getFeatureVector(int unitIndex) { + return featureVectors[unitIndex]; + } + + /** + * Return a shallow copy of the array of feature vectors. + * + * @return a new array containing the internal feature vectors + */ + public FeatureVector[] getCopyOfFeatureVectors() { + return (FeatureVector[]) featureVectors.clone(); + } + + /** + * Return the internal array of feature vectors. + * + * @return the internal array of feature vectors. + */ + public FeatureVector[] getFeatureVectors() { + return featureVectors; + } + + /** + * feature vector mapping according to new feature definition Note: The new feature definition should be a subset of original + * feature definition + * + * @param newFeatureDefinition + * @return + */ + public FeatureVector[] featureVectorMapping(FeatureDefinition newFeatureDefinition) { + + if (!this.featureDefinition.contains(newFeatureDefinition)) { + throw new RuntimeException("the new feature definition is not a subset of original feature definition"); + } + + int numberOfFeatures = newFeatureDefinition.getNumberOfFeatures(); + int noByteFeatures = newFeatureDefinition.getNumberOfByteFeatures(); + int noShortFeatures = newFeatureDefinition.getNumberOfShortFeatures(); + int noContiniousFeatures = newFeatureDefinition.getNumberOfContinuousFeatures(); + + if (numberOfFeatures != (noByteFeatures + noShortFeatures + noContiniousFeatures)) { + throw new RuntimeException("The sum of byte, short and continious features are not equal to number of features"); + } + + String[] featureNames = new String[numberOfFeatures]; + for (int j = 0; j < numberOfFeatures; j++) { + featureNames[j] = newFeatureDefinition.getFeatureName(j); + } + int[] featureIndexes = featureDefinition.getFeatureIndexArray(featureNames); + FeatureVector[] newFV = new FeatureVector[this.getNumberOfUnits()]; + + for (int i = 0; i < this.getNumberOfUnits(); i++) { + + // create features array + byte[] byteFeatures = new byte[noByteFeatures]; + short[] shortFeatures = new short[noShortFeatures]; + float[] continiousFeatures = new float[noContiniousFeatures]; + + int countByteFeatures = 0; + int countShortFeatures = 0; + int countFloatFeatures = 0; + + for (int j = 0; j < featureIndexes.length; j++) { + if (newFeatureDefinition.isByteFeature(j)) { + byteFeatures[countByteFeatures++] = featureVectors[i].getByteFeature(featureIndexes[j]); + } else if (newFeatureDefinition.isShortFeature(j)) { + shortFeatures[countShortFeatures++] = featureVectors[i].getShortFeature(featureIndexes[j]); + } else if (newFeatureDefinition.isContinuousFeature(j)) { + continiousFeatures[countFloatFeatures++] = featureVectors[i].getContinuousFeature(featureIndexes[j]); + } + } + + newFV[i] = newFeatureDefinition.toFeatureVector(i, byteFeatures, shortFeatures, continiousFeatures); + } + + return newFV; + } + + public FeatureDefinition getFeatureDefinition() { + return featureDefinition; + } + + public int getNumberOfUnits() { + return (featureVectors.length); + } +} diff --git a/marytts-runtime/src/main/java/marytts/features/HalfPhoneFeatureFileReader.java b/marytts-runtime/src/main/java/marytts/features/HalfPhoneFeatureFileReader.java new file mode 100644 index 0000000000..b98fb51716 --- /dev/null +++ b/marytts-runtime/src/main/java/marytts/features/HalfPhoneFeatureFileReader.java @@ -0,0 +1,99 @@ +/** + * Copyright 2006 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.features; + +import java.io.BufferedInputStream; +import java.io.DataInputStream; +import java.io.FileInputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; + +import marytts.exceptions.MaryConfigurationException; +import marytts.features.FeatureDefinition; +import marytts.features.FeatureVector; +import marytts.util.data.MaryHeader; + +public class HalfPhoneFeatureFileReader extends FeatureFileReader { + protected FeatureDefinition leftWeights; + protected FeatureDefinition rightWeights; + + public HalfPhoneFeatureFileReader() { + super(); + } + + public HalfPhoneFeatureFileReader(String fileName) throws IOException, MaryConfigurationException { + super(fileName); + } + + @Override + protected void loadFromStream(String fileName) throws IOException, MaryConfigurationException { + /* Open the file */ + DataInputStream dis = null; + dis = new DataInputStream(new BufferedInputStream(new FileInputStream(fileName))); + /* Load the Mary header */ + hdr = new MaryHeader(dis); + if (hdr.getType() != MaryHeader.HALFPHONE_UNITFEATS) { + throw new IOException("File [" + fileName + "] is not a valid Mary Halfphone Features file."); + } + leftWeights = new FeatureDefinition(dis); + rightWeights = new FeatureDefinition(dis); + assert leftWeights.featureEquals(rightWeights) : "Halfphone unit feature file contains incompatible feature definitions for left and right units -- this should not happen!"; + featureDefinition = leftWeights; // one of them, for super class + int numberOfUnits = dis.readInt(); + featureVectors = new FeatureVector[numberOfUnits]; + for (int i = 0; i < numberOfUnits; i++) { + featureVectors[i] = featureDefinition.readFeatureVector(i, dis); + } + } + + @Override + protected void loadFromByteBuffer(String fileName) throws IOException, MaryConfigurationException { + /* Open the file */ + FileInputStream fis = new FileInputStream(fileName); + FileChannel fc = fis.getChannel(); + ByteBuffer bb = fc.map(FileChannel.MapMode.READ_ONLY, 0, fc.size()); + fis.close(); + + /* Load the Mary header */ + hdr = new MaryHeader(bb); + if (hdr.getType() != MaryHeader.HALFPHONE_UNITFEATS) { + throw new MaryConfigurationException("File [" + fileName + "] is not a valid Mary Halfphone Features file."); + } + leftWeights = new FeatureDefinition(bb); + rightWeights = new FeatureDefinition(bb); + assert leftWeights.featureEquals(rightWeights) : "Halfphone unit feature file contains incompatible feature definitions for left and right units -- this should not happen!"; + featureDefinition = leftWeights; // one of them, for super class + int numberOfUnits = bb.getInt(); + featureVectors = new FeatureVector[numberOfUnits]; + for (int i = 0; i < numberOfUnits; i++) { + featureVectors[i] = featureDefinition.readFeatureVector(i, bb); + } + } + + public FeatureDefinition getLeftWeights() { + return leftWeights; + } + + public FeatureDefinition getRightWeights() { + return rightWeights; + } + +} diff --git a/marytts-runtime/src/main/java/marytts/features/HalfPhoneTarget.java b/marytts-runtime/src/main/java/marytts/features/HalfPhoneTarget.java new file mode 100644 index 0000000000..8e29ea8d3f --- /dev/null +++ b/marytts-runtime/src/main/java/marytts/features/HalfPhoneTarget.java @@ -0,0 +1,60 @@ +/** + * Copyright 2006 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.features; + +import org.w3c.dom.Element; + +public class HalfPhoneTarget extends Target { + protected boolean isLeftHalf; + + /** + * Create a target associated to the given segment item. + * + * @param name + * a name for the target, which may or may not coincide with the segment name. + * @param item + * the phone segment item in the Utterance structure, to be associated to this target + * @param isLeftHalf + * true if this target represents the left half of the phone, false if it represents the right half of the phone + */ + public HalfPhoneTarget(String name, Element maryxmlElement, boolean isLeftHalf) { + super(name, maryxmlElement); + this.isLeftHalf = isLeftHalf; + } + + /** + * Is this target the left half of a phone? + * + * @return + */ + public boolean isLeftHalf() { + return isLeftHalf; + } + + /** + * Is this target the right half of a phone? + * + * @return + */ + public boolean isRightHalf() { + return !isLeftHalf; + } + +} diff --git a/marytts-runtime/src/main/java/marytts/features/Target.java b/marytts-runtime/src/main/java/marytts/features/Target.java new file mode 100644 index 0000000000..25486764dd --- /dev/null +++ b/marytts-runtime/src/main/java/marytts/features/Target.java @@ -0,0 +1,191 @@ +/** + * Copyright 2006 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.features; + +import marytts.datatypes.MaryXML; +import marytts.features.FeatureVector; +import marytts.features.MaryGenericFeatureProcessors; +import marytts.modules.phonemiser.Allophone; +import marytts.modules.phonemiser.AllophoneSet; +import marytts.modules.synthesis.Voice; +import marytts.util.MaryRuntimeUtils; +import marytts.util.dom.MaryDomUtils; + +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.UserDataHandler; + +/** + * A representation of a target representing the ideal properties of a unit in a target utterance. + * + * @author Marc Schröder + * + */ +public class Target { + protected String name; + protected Element maryxmlElement; + + protected FeatureVector featureVector = null; + + protected float duration = -1; + protected float f0 = -1; + protected int isSilence = -1; + + /** + * Create a target associated to the given element in the MaryXML tree. + * + * @param name + * a name for the target, which may or may not coincide with the segment name. + * @param maryxmlElement + * the phone or boundary element in the MaryXML tree to be associated with this target. + */ + public Target(String name, Element maryxmlElement) { + this.name = name; + this.maryxmlElement = maryxmlElement; + } + + public Element getMaryxmlElement() { + return maryxmlElement; + } + + public String getName() { + return name; + } + + public FeatureVector getFeatureVector() { + return featureVector; + } + + public void setFeatureVector(FeatureVector featureVector) { + this.featureVector = featureVector; + } + + public float getTargetDurationInSeconds() { + if (duration != -1) { + return duration; + } else { + if (maryxmlElement == null) + return 0; + // throw new NullPointerException("Target "+name+" does not have a maryxml element."); + duration = new MaryGenericFeatureProcessors.UnitDuration().process(this); + return duration; + } + } + + /** + * adapted from {@link MaryGenericFeatureProcessors.UnitDuration#process} + * + * @param newDuration + */ + public void setTargetDurationInSeconds(float newDuration) { + if (maryxmlElement != null) { + if (maryxmlElement.getTagName().equals(MaryXML.PHONE)) { + maryxmlElement.setAttribute("d", Float.toString(newDuration)); + } else { + assert maryxmlElement.getTagName().equals(MaryXML.BOUNDARY) : "segment should be a phone or a boundary, but is a " + + maryxmlElement.getTagName(); + maryxmlElement.setAttribute("duration", Float.toString(newDuration)); + } + } + } + + public float getTargetF0InHz() { + if (f0 != -1) { + return f0; + } else { + if (maryxmlElement == null) + throw new NullPointerException("Target " + name + " does not have a maryxml element."); + float logf0 = new MaryGenericFeatureProcessors.UnitLogF0().process(this); + if (logf0 == 0) + f0 = 0; + else + f0 = (float) Math.exp(logf0); + return f0; + } + } + + public boolean hasFeatureVector() { + return featureVector != null; + } + + public static UserDataHandler targetFeatureCloner = new UserDataHandler() { + public void handle(short operation, String key, Object data, Node src, Node dest) { + if (operation == UserDataHandler.NODE_CLONED && key == "target") { + dest.setUserData(key, data, this); + System.err.println("yay"); + } else { + System.err.println("nay"); + } + } + }; + + /** + * Determine whether this target is a silence target + * + * @return true if the target represents silence, false otherwise + */ + public boolean isSilence() { + + if (isSilence == -1) { + // TODO: how do we know the silence symbol here? + String silenceSymbol = "_"; + if (name.startsWith(silenceSymbol)) { + isSilence = 1; // true + } else { + isSilence = 0; // false + } + } + return isSilence == 1; + } + + public Allophone getAllophone() { + if (maryxmlElement != null) { + AllophoneSet allophoneSet = null; + Element voiceElement = (Element) MaryDomUtils.getAncestor(maryxmlElement, MaryXML.VOICE); + if (voiceElement != null) { + Voice v = Voice.getVoice(voiceElement); + if (v != null) { + allophoneSet = v.getAllophoneSet(); + } + } + if (allophoneSet == null) { + try { + allophoneSet = MaryRuntimeUtils.determineAllophoneSet(maryxmlElement); + } catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + String sampa; + if (maryxmlElement.getNodeName().equals(MaryXML.PHONE)) { + sampa = maryxmlElement.getAttribute("p"); + } else { + assert maryxmlElement.getNodeName().equals(MaryXML.BOUNDARY); + sampa = "_"; + } + return allophoneSet.getAllophone(sampa); + } + return null; + } + + public String toString() { + return name; + } +} diff --git a/marytts-unitselection/src/main/java/marytts/cart/CART.java b/marytts-unitselection/src/main/java/marytts/cart/CART.java new file mode 100644 index 0000000000..ac65c1ad85 --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/cart/CART.java @@ -0,0 +1,177 @@ +/** + * Portions Copyright 2006 DFKI GmbH. + * Portions Copyright 2001 Sun Microsystems, Inc. + * Portions Copyright 1999-2001 Language Technologies Institute, + * Carnegie Mellon University. + * All Rights Reserved. Use is subject to license terms. + * + * Permission is hereby granted, free of charge, to use and distribute + * this software and its documentation without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of this work, and to + * permit persons to whom this work is furnished to do so, subject to + * the following conditions: + * + * 1. The code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * 2. Any modifications must be clearly marked as such. + * 3. Original authors' names are not deleted. + * 4. The authors' names are not used to endorse or promote products + * derived from this software without specific prior written + * permission. + * + * DFKI GMBH AND THE CONTRIBUTORS TO THIS WORK DISCLAIM ALL WARRANTIES WITH + * REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL DFKI GMBH NOR THE + * CONTRIBUTORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL + * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR + * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS + * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF + * THIS SOFTWARE. + */ +package marytts.cart; + +import java.util.Properties; + +import marytts.features.FeatureDefinition; +import marytts.features.FeatureVector; +import marytts.features.Target; + +/** + * A tree is a specific kind of directed graph in which each node can have only a single parent node. It consists exclusively of + * DecisionNode and LeafNode nodes. + * + * @author marc + * + */ +public class CART extends DirectedGraph { + + /** + * Build a new empty cart + * + */ + public CART() { + } + + /** + * Build a new empty cart with the given feature definition. + * + * @param featDef + */ + public CART(FeatureDefinition featDef) { + super(featDef); + } + + /** + * Build a new cart with the given node as the root node + * + * @param rootNode + * the root node of the CART + * @param featDef + * the feature definition used for interpreting the meaning of decision node criteria. + */ + public CART(Node rootNode, FeatureDefinition featDef) { + super(rootNode, featDef); + } + + /** + * Build a new cart with the given node as the root node + * + * @param rootNode + * the root node of the CART + * @param featDef + * the feature definition used for interpreting the meaning of decision node criteria. + * @param properties + * a generic properties object, which can be used to encode information about the tree and the way the data in it + * should be represented. + */ + public CART(Node rootNode, FeatureDefinition featDef, Properties properties) { + super(rootNode, featDef, properties); + } + + /** + * Passes the given item through this CART and returns the leaf Node, or the Node it stopped walking down. + * + * @param target + * the target to analyze + * @param minNumberOfData + * the minimum number of data requested. If this is 0, walk down the CART until the leaf level. + * + * @return the Node + */ + public Node interpretToNode(Target target, int minNumberOfData) { + return interpretToNode(target.getFeatureVector(), minNumberOfData); + } + + /** + * Passes the given item through this CART and returns the leaf Node, or the Node it stopped walking down. + * + * @param target + * the target to analyze + * @param minNumberOfData + * the minimum number of data requested. If this is 0, walk down the CART until the leaf level. + * + * @return the Node + */ + public Node interpretToNode(FeatureVector featureVector, int minNumberOfData) { + Node currentNode = rootNode; + Node prevNode = null; + + // logger.debug("Starting cart at "+nodeIndex); + while (currentNode != null && currentNode.getNumberOfData() > minNumberOfData && !(currentNode instanceof LeafNode)) { + // while we have not reached the bottom, + // get the next node based on the features of the target + prevNode = currentNode; + currentNode = ((DecisionNode) currentNode).getNextNode(featureVector); + // logger.debug(decision.toString() + " result '"+ + // decision.findFeature(item) + "' => "+ nodeIndex); + } + // Now usually we will have gone down one level too far + if (currentNode == null || currentNode.getNumberOfData() < minNumberOfData && prevNode != null) { + currentNode = prevNode; + } + + assert currentNode.getNumberOfData() >= minNumberOfData || currentNode == rootNode; + + assert minNumberOfData > 0 || (currentNode instanceof LeafNode); + return currentNode; + + } + + /** + * Passes the given item through this CART and returns the interpretation. + * + * @param target + * the target to analyze + * @param minNumberOfData + * the minimum number of data requested. If this is 0, walk down the CART until the leaf level. + * + * @return the interpretation + */ + public Object interpret(Target target, int minNumberOfData) { + + // get the indices from the leaf node + Object result = this.interpretToNode(target, minNumberOfData).getAllData(); + + return result; + + } + + /** + * In this tree, replace the given leaf with the given CART + * + * @param cart + * the CART + * @param leaf + * the leaf + * @return the ex-root node from cart which now replaces leaf. + */ + public static Node replaceLeafByCart(CART cart, LeafNode leaf) { + DecisionNode mother = (DecisionNode) leaf.getMother(); + Node newNode = cart.getRootNode(); + mother.replaceDaughter(newNode, leaf.getNodeIndex()); + newNode.setIsRoot(false); + return newNode; + } + +} diff --git a/marytts-unitselection/src/main/java/marytts/cart/DecisionNode.java b/marytts-unitselection/src/main/java/marytts/cart/DecisionNode.java new file mode 100644 index 0000000000..8cb0b8506c --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/cart/DecisionNode.java @@ -0,0 +1,748 @@ +/** + * Copyright 2000-2009 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.cart; + +import marytts.cart.LeafNode.FeatureVectorLeafNode; +import marytts.cart.LeafNode.IntArrayLeafNode; +import marytts.features.FeatureDefinition; +import marytts.features.FeatureVector; + +/** + * A decision node that determines the next Node to go to in the CART. All decision nodes inherit from this class + */ +public abstract class DecisionNode extends Node { + public enum Type { + BinaryByteDecisionNode, BinaryShortDecisionNode, BinaryFloatDecisionNode, ByteDecisionNode, ShortDecisionNode + }; + + protected boolean TRACE = false; + // for debugging: + protected FeatureDefinition featureDefinition; + + // a decision node has an array of daughters + protected Node[] daughters; + + // the feature index + protected int featureIndex; + + // the feature name + protected String feature; + + // remember last added daughter + protected int lastDaughter; + + // the total number of data in the leaves below this node + protected int nData; + + // unique index used in MaryCART format + protected int uniqueDecisionNodeId; + + /** + * Construct a new DecisionNode + * + * @param feature + * the feature + * @param numDaughters + * the number of daughters + */ + public DecisionNode(String feature, int numDaughters, FeatureDefinition featureDefinition) { + this.feature = feature; + this.featureIndex = featureDefinition.getFeatureIndex(feature); + daughters = new Node[numDaughters]; + isRoot = false; + // for trace and getDecisionPath(): + this.featureDefinition = featureDefinition; + } + + /** + * Construct a new DecisionNode + * + * @param featureIndex + * the feature index + * @param numDaughters + * the number of daughters + */ + public DecisionNode(int featureIndex, int numDaughters, FeatureDefinition featureDefinition) { + this.featureIndex = featureIndex; + this.feature = featureDefinition.getFeatureName(featureIndex); + daughters = new Node[numDaughters]; + isRoot = false; + // for trace and getDecisionPath(): + this.featureDefinition = featureDefinition; + } + + /** + * Construct a new DecisionNode + * + * @param numDaughters + * the number of daughters + */ + public DecisionNode(int numDaughters, FeatureDefinition featureDefinition) { + daughters = new Node[numDaughters]; + isRoot = false; + // for trace and getDecisionPath(): + this.featureDefinition = featureDefinition; + } + + @Override + public boolean isDecisionNode() { + return true; + } + + /** + * Get the name of the feature + * + * @return the name of the feature + */ + public String getFeatureName() { + return feature; + } + + public int getFeatureIndex() { + return featureIndex; + } + + public FeatureDefinition getFeatureDefinition() { + return featureDefinition; + } + + /** + * Add a daughter to the node + * + * @param daughter + * the new daughter + */ + public void addDaughter(Node daughter) { + if (lastDaughter > daughters.length - 1) { + throw new RuntimeException("Can not add daughter number " + (lastDaughter + 1) + ", since node has only " + + daughters.length + " daughters!"); + } + daughters[lastDaughter] = daughter; + if (daughter != null) { + daughter.setMother(this, lastDaughter); + } + lastDaughter++; + } + + /** + * Get the daughter at the specified index + * + * @param index + * the index of the daughter + * @return the daughter (potentially null); if index out of range: null + */ + public Node getDaughter(int index) { + if (index > daughters.length - 1 || index < 0) { + return null; + } + return daughters[index]; + } + + /** + * Replace daughter at given index with another daughter + * + * @param newDaughter + * the new daughter + * @param index + * the index of the daughter to replace + */ + public void replaceDaughter(Node newDaughter, int index) { + if (index > daughters.length - 1 || index < 0) { + throw new RuntimeException("Can not replace daughter number " + index + ", since daughter index goes from 0 to " + + (daughters.length - 1) + "!"); + } + daughters[index] = newDaughter; + newDaughter.setMother(this, index); + } + + /** + * Tests, if the given index refers to a daughter + * + * @param index + * the index + * @return true, if the index is in range of the daughters array + */ + public boolean hasMoreDaughters(int index) { + return (index > -1 && index < daughters.length); + } + + /** + * Get all unit indices from all leaves below this node + * + * @return an int array containing the indices + */ + public Object getAllData() { + // What to do depends on the type of leaves. + LeafNode firstLeaf = new NodeIterator(this, true, false, false).next(); + if (firstLeaf == null) + return null; + Object result; + if (firstLeaf instanceof IntArrayLeafNode) { // this includes subclass IntAndFloatArrayLeafNode + result = new int[nData]; + } else if (firstLeaf instanceof FeatureVectorLeafNode) { + result = new FeatureVector[nData]; + } else { + return null; + } + fillData(result, 0, nData); + return result; + } + + protected void fillData(Object target, int pos, int total) { + // assert pos + total <= target.length; + for (int i = 0; i < daughters.length; i++) { + if (daughters[i] == null) + continue; + int len = daughters[i].getNumberOfData(); + daughters[i].fillData(target, pos, len); + pos += len; + } + } + + /** + * Count all the nodes at and below this node. A leaf will return 1; the root node will report the total number of decision + * and leaf nodes in the tree. + * + * @return + */ + public int getNumberOfNodes() { + int nNodes = 1; // this node + for (int i = 0; i < daughters.length; i++) { + if (daughters[i] != null) + nNodes += daughters[i].getNumberOfNodes(); + } + return nNodes; + } + + public int getNumberOfData() { + return nData; + } + + /** Number of daughters of current node. */ + public int getNumberOfDaugthers() { + return daughters.length; + } + + /** + * Set the number of candidates correctly, by counting while walking down the tree. This needs to be done once for the entire + * tree. + * + */ + // protected void countData() { + public void countData() { + nData = 0; + for (int i = 0; i < daughters.length; i++) { + if (daughters[i] instanceof DecisionNode) + ((DecisionNode) daughters[i]).countData(); + if (daughters[i] != null) { + nData += daughters[i].getNumberOfData(); + } + } + } + + public String toString() { + return "dn" + uniqueDecisionNodeId; + } + + /** + * Get the path leading to the daughter with the given index. This will recursively go up to the root node. + * + * @param daughterIndex + * @return + */ + public abstract String getDecisionPath(int daughterIndex); + + // unique index used in MaryCART format + public void setUniqueDecisionNodeId(int id) { + this.uniqueDecisionNodeId = id; + } + + public int getUniqueDecisionNodeId() { + return uniqueDecisionNodeId; + } + + /** + * Gets the String that defines the decision done in the node + * + * @return the node definition + */ + public abstract String getNodeDefinition(); + + /** + * Get the decision node type + * + * @return + */ + public abstract Type getDecisionNodeType(); + + /** + * Select a daughter node according to the value in the given target + * + * @param target + * the target + * @return a daughter + */ + public abstract Node getNextNode(FeatureVector featureVector); + + /** + * A binary decision Node that compares two byte values. + */ + public static class BinaryByteDecisionNode extends DecisionNode { + + // the value of this node + private byte value; + + /** + * Create a new binary String DecisionNode. + * + * @param feature + * the string used to get a value from an Item + * @param value + * the value to compare to + */ + public BinaryByteDecisionNode(String feature, String value, FeatureDefinition featureDefinition) { + super(feature, 2, featureDefinition); + this.value = featureDefinition.getFeatureValueAsByte(feature, value); + } + + public BinaryByteDecisionNode(int featureIndex, byte value, FeatureDefinition featureDefinition) { + super(featureIndex, 2, featureDefinition); + this.value = value; + } + + /*** + * Creates an empty BinaryByteDecisionNode, the feature and feature value of this node should be filled with + * setFeatureAndFeatureValue() function. + * + * @param uniqueId + * unique index from tree HTS test file. + * @param featureDefinition + */ + public BinaryByteDecisionNode(int uniqueId, FeatureDefinition featureDefinition) { + super(2, featureDefinition); + // System.out.println("adding decision node: " + uniqueId); + this.uniqueDecisionNodeId = uniqueId; + } + + /*** + * Fill the feature and feature value of an already created (empty) BinaryByteDecisionNode. + * + * @param feature + * @param value + */ + public void setFeatureAndFeatureValue(String feature, String value) { + this.feature = feature; + this.featureIndex = featureDefinition.getFeatureIndex(feature); + this.value = featureDefinition.getFeatureValueAsByte(feature, value); + } + + public byte getCriterionValueAsByte() { + return value; + } + + public String getCriterionValueAsString() { + return featureDefinition.getFeatureValueAsString(featureIndex, value); + } + + /** + * Select a daughter node according to the value in the given target + * + * @param target + * the target + * @return a daughter + */ + public Node getNextNode(FeatureVector featureVector) { + byte val = featureVector.getByteFeature(featureIndex); + Node returnNode; + if (val == value) { + returnNode = daughters[0]; + } else { + returnNode = daughters[1]; + } + if (TRACE) { + System.out.print(" " + feature + ": " + featureDefinition.getFeatureValueAsString(featureIndex, value) + + " == " + featureDefinition.getFeatureValueAsString(featureIndex, val)); + if (val == value) + System.out.println(" YES "); + else + System.out.println(" NO "); + } + return returnNode; + } + + public String getDecisionPath(int daughterIndex) { + String thisNodeInfo; + if (daughterIndex == 0) + thisNodeInfo = feature + "==" + featureDefinition.getFeatureValueAsString(featureIndex, value); + else + thisNodeInfo = feature + "!=" + featureDefinition.getFeatureValueAsString(featureIndex, value); + if (mother == null) + return thisNodeInfo; + else if (mother.isDecisionNode()) + return ((DecisionNode) mother).getDecisionPath(getNodeIndex()) + " - " + thisNodeInfo; + else + return mother.getDecisionPath() + " - " + thisNodeInfo; + } + + /** + * Gets the String that defines the decision done in the node + * + * @return the node definition + */ + public String getNodeDefinition() { + return feature + " is " + featureDefinition.getFeatureValueAsString(featureIndex, value); + } + + public Type getDecisionNodeType() { + return Type.BinaryByteDecisionNode; + } + + } + + /** + * A binary decision Node that compares two short values. + */ + public static class BinaryShortDecisionNode extends DecisionNode { + + // the value of this node + private short value; + + /** + * Create a new binary String DecisionNode. + * + * @param feature + * the string used to get a value from an Item + * @param value + * the value to compare to + */ + public BinaryShortDecisionNode(String feature, String value, FeatureDefinition featureDefinition) { + super(feature, 2, featureDefinition); + this.value = featureDefinition.getFeatureValueAsShort(feature, value); + } + + public BinaryShortDecisionNode(int featureIndex, short value, FeatureDefinition featureDefinition) { + super(featureIndex, 2, featureDefinition); + this.value = value; + } + + public short getCriterionValueAsShort() { + return value; + } + + public String getCriterionValueAsString() { + return featureDefinition.getFeatureValueAsString(featureIndex, value); + } + + /** + * Select a daughter node according to the value in the given target + * + * @param target + * the target + * @return a daughter + */ + public Node getNextNode(FeatureVector featureVector) { + short val = featureVector.getShortFeature(featureIndex); + Node returnNode; + if (val == value) { + returnNode = daughters[0]; + } else { + returnNode = daughters[1]; + } + if (TRACE) { + System.out.print(feature + ": " + featureDefinition.getFeatureValueAsString(featureIndex, val)); + if (val == value) + System.out.print(" == "); + else + System.out.print(" != "); + System.out.println(featureDefinition.getFeatureValueAsString(featureIndex, value)); + } + return returnNode; + } + + public String getDecisionPath(int daughterIndex) { + String thisNodeInfo; + if (daughterIndex == 0) + thisNodeInfo = feature + "==" + featureDefinition.getFeatureValueAsString(featureIndex, value); + else + thisNodeInfo = feature + "!=" + featureDefinition.getFeatureValueAsString(featureIndex, value); + if (mother == null) + return thisNodeInfo; + else if (mother.isDecisionNode()) + return ((DecisionNode) mother).getDecisionPath(getNodeIndex()) + " - " + thisNodeInfo; + else + return mother.getDecisionPath() + " - " + thisNodeInfo; + } + + /** + * Gets the String that defines the decision done in the node + * + * @return the node definition + */ + public String getNodeDefinition() { + return feature + " is " + featureDefinition.getFeatureValueAsString(featureIndex, value); + } + + public Type getDecisionNodeType() { + return Type.BinaryShortDecisionNode; + } + + } + + /** + * A binary decision Node that compares two float values. + */ + public static class BinaryFloatDecisionNode extends DecisionNode { + + // the value of this node + private float value; + private boolean isByteFeature; + + /** + * Create a new binary String DecisionNode. + * + * @param feature + * the string used to get a value from an Item + * @param value + * the value to compare to + */ + public BinaryFloatDecisionNode(int featureIndex, float value, FeatureDefinition featureDefinition) { + this(featureDefinition.getFeatureName(featureIndex), value, featureDefinition); + } + + public BinaryFloatDecisionNode(String feature, float value, FeatureDefinition featureDefinition) { + super(feature, 2, featureDefinition); + this.value = value; + // check for pseudo-floats: + // TODO: clean this up: + if (featureDefinition.isByteFeature(featureIndex)) + isByteFeature = true; + else + isByteFeature = false; + } + + public float getCriterionValueAsFloat() { + return value; + } + + public String getCriterionValueAsString() { + return String.valueOf(value); + } + + /** + * Select a daughter node according to the value in the given target + * + * @param target + * the target + * @return a daughter + */ + public Node getNextNode(FeatureVector featureVector) { + float val; + if (isByteFeature) + val = (float) featureVector.getByteFeature(featureIndex); + else + val = featureVector.getContinuousFeature(featureIndex); + Node returnNode; + if (val < value) { + returnNode = daughters[0]; + } else { + returnNode = daughters[1]; + } + if (TRACE) { + System.out.print(feature + ": " + val); + if (val < value) + System.out.print(" < "); + else + System.out.print(" >= "); + System.out.println(value); + } + + return returnNode; + } + + public String getDecisionPath(int daughterIndex) { + String thisNodeInfo; + if (daughterIndex == 0) + thisNodeInfo = feature + "<" + value; + else + thisNodeInfo = feature + ">=" + value; + if (mother == null) + return thisNodeInfo; + else if (mother.isDecisionNode()) + return ((DecisionNode) mother).getDecisionPath(getNodeIndex()) + " - " + thisNodeInfo; + else + return mother.getDecisionPath() + " - " + thisNodeInfo; + } + + /** + * Gets the String that defines the decision done in the node + * + * @return the node definition + */ + public String getNodeDefinition() { + return feature + " < " + value; + } + + public Type getDecisionNodeType() { + return Type.BinaryFloatDecisionNode; + } + + } + + /** + * An decision Node with an arbitrary number of daughters. Value of the target corresponds to the index number of next + * daughter. + */ + public static class ByteDecisionNode extends DecisionNode { + + /** + * Build a new byte decision node + * + * @param feature + * the feature name + * @param numDaughters + * the number of daughters + */ + public ByteDecisionNode(String feature, int numDaughters, FeatureDefinition featureDefinition) { + super(feature, numDaughters, featureDefinition); + } + + /** + * Build a new byte decision node + * + * @param feature + * the feature name + * @param numDaughters + * the number of daughters + */ + public ByteDecisionNode(int featureIndex, int numDaughters, FeatureDefinition featureDefinition) { + super(featureIndex, numDaughters, featureDefinition); + } + + /** + * Select a daughter node according to the value in the given target + * + * @param target + * the target + * @return a daughter + */ + public Node getNextNode(FeatureVector featureVector) { + byte val = featureVector.getByteFeature(featureIndex); + if (TRACE) { + System.out.println(feature + ": " + featureDefinition.getFeatureValueAsString(featureIndex, val)); + } + return daughters[val]; + } + + public String getDecisionPath(int daughterIndex) { + String thisNodeInfo = feature + "==" + featureDefinition.getFeatureValueAsString(featureIndex, daughterIndex); + if (mother == null) + return thisNodeInfo; + else if (mother.isDecisionNode()) + return ((DecisionNode) mother).getDecisionPath(getNodeIndex()) + " - " + thisNodeInfo; + else + return mother.getDecisionPath() + " - " + thisNodeInfo; + } + + /** + * Gets the String that defines the decision done in the node + * + * @return the node definition + */ + public String getNodeDefinition() { + return feature + " isByteOf " + daughters.length; + } + + public Type getDecisionNodeType() { + return Type.ByteDecisionNode; + } + + } + + /** + * An decision Node with an arbitrary number of daughters. Value of the target corresponds to the index number of next + * daughter. + */ + public static class ShortDecisionNode extends DecisionNode { + + /** + * Build a new short decision node + * + * @param feature + * the feature name + * @param numDaughters + * the number of daughters + */ + public ShortDecisionNode(String feature, int numDaughters, FeatureDefinition featureDefinition) { + super(feature, numDaughters, featureDefinition); + } + + /** + * Build a new short decision node + * + * @param featureIndex + * the feature index + * @param numDaughters + * the number of daughters + */ + public ShortDecisionNode(int featureIndex, int numDaughters, FeatureDefinition featureDefinition) { + super(featureIndex, numDaughters, featureDefinition); + } + + /** + * Select a daughter node according to the value in the given target + * + * @param target + * the target + * @return a daughter + */ + public Node getNextNode(FeatureVector featureVector) { + short val = featureVector.getShortFeature(featureIndex); + if (TRACE) { + System.out.println(feature + ": " + featureDefinition.getFeatureValueAsString(featureIndex, val)); + } + return daughters[val]; + } + + public String getDecisionPath(int daughterIndex) { + String thisNodeInfo = feature + "==" + featureDefinition.getFeatureValueAsString(featureIndex, daughterIndex); + if (mother == null) + return thisNodeInfo; + else if (mother.isDecisionNode()) + return ((DecisionNode) mother).getDecisionPath(getNodeIndex()) + " - " + thisNodeInfo; + else + return mother.getDecisionPath() + " - " + thisNodeInfo; + } + + /** + * Gets the String that defines the decision done in the node + * + * @return the node definition + */ + public String getNodeDefinition() { + return feature + " isShortOf " + daughters.length; + } + + public Type getDecisionNodeType() { + return Type.ShortDecisionNode; + } + + } + +} diff --git a/marytts-unitselection/src/main/java/marytts/cart/DirectedGraph.java b/marytts-unitselection/src/main/java/marytts/cart/DirectedGraph.java new file mode 100644 index 0000000000..3d3e1db64e --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/cart/DirectedGraph.java @@ -0,0 +1,276 @@ +/** + * Copyright 2009 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ + +package marytts.cart; + +import java.util.Iterator; +import java.util.Properties; + +import marytts.features.FeatureDefinition; +import marytts.features.FeatureVector; +import marytts.features.Target; +import marytts.util.MaryUtils; + +import org.apache.log4j.Logger; + +/** + * A directed graph is a layered structure of nodes, in which there are mother-daughter relationships between the node. There is a + * single root node. Each node can have multiple daughters and/or multiple mothers. Three types of nodes are allowed: + * DirectedGraphNode (which can have multiple mothers, a leaf and a decision node), LeafNodes (which carry data), and + * DecisionNodes (which can have multiple daughters). + * + * @author marc + * + */ +public class DirectedGraph { + protected Logger logger = MaryUtils.getLogger(this.getClass().getName()); + + protected Node rootNode; + + // knows the index numbers and types of the features used in DecisionNodes + protected FeatureDefinition featDef; + + protected Properties properties; + + /** + * Build a new empty directed graph + * + */ + public DirectedGraph() { + } + + /** + * Build a new empty graph with the given feature definition. + * + * @param featDef + * the feature definition used for interpreting the meaning of decision node criteria. + */ + public DirectedGraph(FeatureDefinition featDef) { + this(null, featDef); + } + + /** + * Build a new graph with the given node as the root node + * + * @param rootNode + * the root node of the graph + * @param featDef + * the feature definition used for interpreting the meaning of decision node criteria. + */ + public DirectedGraph(Node rootNode, FeatureDefinition featDef) { + this(rootNode, featDef, null); + } + + /** + * Build a new graph with the given node as the root node + * + * @param rootNode + * the root node of the graph + * @param featDef + * the feature definition used for interpreting the meaning of decision node criteria. + * @param properties + * a generic properties object, which can be used to encode information about the tree and the way the data in it + * should be represented. + */ + public DirectedGraph(Node rootNode, FeatureDefinition featDef, Properties properties) { + this.rootNode = rootNode; + this.featDef = featDef; + this.properties = properties; + } + + public Object interpret(Target t) { + return interpret(t.getFeatureVector()); + } + + /** + * Walk down the graph as far as possible according to the features in fv, and return the data in the leaf node found there. + * + * @param fv + * a feature vector which must be consistent with the graph's feature definition. (@see #getFeatureDefinition()). + * @return the most specific non-null leaf node data that can be retrieved, or null if there is no non-null leaf node data + * along the fv's path. + */ + public Object interpret(FeatureVector fv) { + return interpret(rootNode, fv); + } + + /** + * Follow the directed graph down to the most specific leaf with data, starting from node n. This is recursively calling + * itself. + * + * @param n + * @param fv + * @return + */ + protected Object interpret(Node n, FeatureVector fv) { + if (n == null) + return null; + else if (n.isLeafNode()) { + return n.getAllData(); + } else if (n.isDecisionNode()) { + Node next = ((DecisionNode) n).getNextNode(fv); + return interpret(next, fv); + } else if (n.isDirectedGraphNode()) { + DirectedGraphNode g = (DirectedGraphNode) n; + Object data = interpret(g.getDecisionNode(), fv); + if (data != null) { // OK, found something more specific + return data; + } + return interpret(g.getLeafNode(), fv); + } + throw new IllegalArgumentException("Unknown node type: " + n.getClass()); + } + + /** + * Return an iterator which returns all nodes in the tree exactly once. Search is done in a depth-first way. + * + * @return + */ + public Iterator getNodeIterator() { + return new NodeIterator(this, true, true, true); + } + + /** + * Return an iterator which returns all leaf nodes in the tree exactly once. Search is done in a depth-first way. + * + * @return + */ + public Iterator getLeafNodeIterator() { + return new NodeIterator(this, true, false, false); + } + + /** + * Return an iterator which returns all decision nodes in the tree exactly once. Search is done in a depth-first way. + * + * @return + */ + public Iterator getDecisionNodeIterator() { + return new NodeIterator(this, false, true, false); + } + + /** + * Return an iterator which returns all directed graph nodes in the tree exactly once. Search is done in a depth-first way. + * + * @return + */ + public Iterator getDirectedGraphNodeIterator() { + return new NodeIterator(this, false, false, true); + } + + /** + * A representation of the corresponding node iterator that can be used in extended for() statements. + * + * @return + */ + public Iterable getNodes() { + return new Iterable() { + public Iterator iterator() { + return getNodeIterator(); + } + }; + } + + /** + * A representation of the corresponding node iterator that can be used in extended for() statements. + * + * @return + */ + public Iterable getLeafNodes() { + return new Iterable() { + public Iterator iterator() { + return getLeafNodeIterator(); + } + }; + } + + /** + * A representation of the corresponding node iterator that can be used in extended for() statements. + * + * @return + */ + public Iterable getDecisionNodes() { + return new Iterable() { + public Iterator iterator() { + return getDecisionNodeIterator(); + } + }; + } + + /** + * A representation of the corresponding node iterator that can be used in extended for() statements. + * + * @return + */ + public Iterable getDirectedGraphNodes() { + return new Iterable() { + public Iterator iterator() { + return getDirectedGraphNodeIterator(); + } + }; + } + + /** + * Get the properties object associated with this tree, or null if there is no such object. + * + * @return + */ + public Properties getProperties() { + return properties; + } + + /** + * Get the root node of this CART + * + * @return the root node + */ + public Node getRootNode() { + return rootNode; + } + + /** + * Set the root node of this CART + * + * @param the + * root node + */ + public void setRootNode(Node rNode) { + rootNode = rNode; + } + + public FeatureDefinition getFeatureDefinition() { + return featDef; + } + + /** + * Get the number of nodes in this CART + * + * @return the number of nodes + */ + public int getNumNodes() { + if (rootNode == null) + return 0; + return rootNode.getNumberOfNodes(); + } + + public String toString() { + return this.rootNode.toString(""); + } + +} diff --git a/marytts-unitselection/src/main/java/marytts/cart/DirectedGraphNode.java b/marytts-unitselection/src/main/java/marytts/cart/DirectedGraphNode.java new file mode 100644 index 0000000000..7190a7e723 --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/cart/DirectedGraphNode.java @@ -0,0 +1,218 @@ +/** + * Copyright 2009 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ + +package marytts.cart; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import marytts.features.FeatureVector; + +/** + * A type of node that can be at the same time a decision node and a leaf node, and that can have more than one mother. Other than + * tree nodes, thus, directed graph nodes are not necessarily contained in a strict tree structure; furthermore, each node can + * potentially carry data. + * + * @author marc + * + */ +public class DirectedGraphNode extends Node { + + private DecisionNode decisionNode; + private Node leafNode; + + private Map motherToIndex = new HashMap(); + private List mothers = new ArrayList(); + private int uniqueID; + + /** + * + */ + public DirectedGraphNode(DecisionNode decisionNode, Node leafNode) { + setDecisionNode(decisionNode); + setLeafNode(leafNode); + } + + public DecisionNode getDecisionNode() { + return decisionNode; + } + + @Override + public boolean isDirectedGraphNode() { + return true; + } + + public void setDecisionNode(DecisionNode newNode) { + this.decisionNode = newNode; + if (newNode != null) + newNode.setMother(this, 0); + } + + public Node getLeafNode() { + return leafNode; + } + + public void setLeafNode(Node newNode) { + if (!(newNode == null || newNode instanceof DirectedGraphNode || newNode instanceof LeafNode)) { + throw new IllegalArgumentException("Only leaf nodes and directed graph nodes allowed as leafNode"); + } + this.leafNode = newNode; + if (newNode != null) + newNode.setMother(this, 0); + } + + @Override + public void setMother(Node node, int nodeIndex) { + mothers.add(node); + motherToIndex.put(node, nodeIndex); + } + + /** + * Get a mother node of this node. DirectedGraphNodes can have more than one node. + * + * @return the first mother, or null if there is no mother. + */ + public Node getMother() { + if (mothers.isEmpty()) + return null; + return mothers.get(0); + } + + /** + * Get the index of this node in the mother returned by getMother(). + * + * @return the index in the mother's daughter array, or 0 if there is no mother. + */ + public int getNodeIndex() { + Node firstMother = getMother(); + if (firstMother != null) + return motherToIndex.get(firstMother); + return 0; + } + + public List getMothers() { + return mothers; + } + + /** + * Return this node's index in the given mother's array of daughters. + * + * @param aMother + * @return + * @throws IllegalArgumentException + * if mother is not a mother of this node. + */ + public int getNodeIndex(Node aMother) { + if (!motherToIndex.containsKey(aMother)) + throw new IllegalArgumentException("The given node is not a mother of this node"); + return motherToIndex.get(aMother); + } + + /** + * Remove the given node from the list of mothers. + * + * @param aMother + * @throws IllegalArgumentException + * if mother is not a mother of this node. + */ + public void removeMother(Node aMother) { + if (!motherToIndex.containsKey(aMother)) + throw new IllegalArgumentException("The given node is not a mother of this node"); + motherToIndex.remove(aMother); + mothers.remove(aMother); + } + + @Override + protected void fillData(Object target, int pos, int len) { + if (leafNode != null) + leafNode.fillData(target, pos, len); + } + + @Override + public Object getAllData() { + if (leafNode != null) + return leafNode.getAllData(); + else if (decisionNode != null) + return decisionNode.getAllData(); + return null; + } + + @Override + public int getNumberOfData() { + if (leafNode != null) + return leafNode.getNumberOfData(); + else if (decisionNode != null) + return decisionNode.getNumberOfData(); + return 0; + } + + @Override + public int getNumberOfNodes() { + if (decisionNode != null) + return decisionNode.getNumberOfNodes(); + return 0; + } + + public Node getNextNode(FeatureVector fv) { + if (decisionNode != null) { + Node next = decisionNode.getNextNode(fv); + if (next != null) + return next; + } + return leafNode; + } + + public int getUniqueGraphNodeID() { + return uniqueID; + } + + public void setUniqueGraphNodeID(int id) { + this.uniqueID = id; + } + + public String getDecisionPath() { + StringBuilder ancestorInfo = new StringBuilder(); + if (getMothers().size() == 0) + ancestorInfo.append("null"); + for (Node mum : getMothers()) { + if (ancestorInfo.length() > 0) { + ancestorInfo.append(" or\n"); + } + if (mum.isDecisionNode()) { + ancestorInfo.append(((DecisionNode) mum).getDecisionPath(getNodeIndex())); + } else { + ancestorInfo.append(mum.getDecisionPath()); + } + } + return ancestorInfo + " - " + toString(); + } + + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("DGN"); + sb.append(uniqueID); + if (motherToIndex.size() > 1) { + sb.append(" (").append(motherToIndex.size()).append(" mothers)"); + } + return sb.toString(); + } +} diff --git a/marytts-unitselection/src/main/java/marytts/cart/FeatureVectorCART.java b/marytts-unitselection/src/main/java/marytts/cart/FeatureVectorCART.java new file mode 100644 index 0000000000..bc7965fcf7 --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/cart/FeatureVectorCART.java @@ -0,0 +1,129 @@ +/** + * Copyright 2007 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.cart; + +import java.io.IOException; + +import marytts.cart.impose.FeatureArrayIndexer; +import marytts.cart.impose.MaryNode; +import marytts.features.FeatureDefinition; +import marytts.features.FeatureVector; + +/** + * @author marc + * + */ +public class FeatureVectorCART extends CART { + + /** + * Convert the given Mary node tree into a CART with the leaves containing featureVectors + * + * @param tree + * the tree + * @param ffi + * the feature file indexer containing the feature vectors + */ + public FeatureVectorCART(MaryNode tree, FeatureArrayIndexer ffi) { + featDef = ffi.getFeatureDefinition(); + addDaughters(null, tree, ffi); + if (rootNode instanceof DecisionNode) { + ((DecisionNode) rootNode).countData(); + } + } + + public void load(String fileName, FeatureDefinition featDefinition, String[] setFeatureSequence) throws IOException { + throw new IllegalStateException("load() not implemented for FeatureVectorCART"); + } + + /** + * Add the given tree node as a daughter to the given mother node + * + * @param motherCARTNode + * the mother node + * @param currentTreeNode + * the tree node that we want to add + * @param ffi + * the feature file indexer containing the feature vectors + */ + private void addDaughters(DecisionNode motherCARTNode, MaryNode currentTreeNode, FeatureArrayIndexer ffi) { + if (currentTreeNode == null) { + LeafNode l = new LeafNode.FeatureVectorLeafNode(new FeatureVector[0]); + motherCARTNode.addDaughter(l); + return; + } + if (currentTreeNode.isNode()) { // if we are not at a leaf + + // System.out.print("Adding node, "); + // the next daughter + DecisionNode daughterNode = null; + // the number of daughters of the next daughter + int numDaughters; + // the index of the next feature + int nextFeatIndex = currentTreeNode.getFeatureIndex(); + // System.out.print("featureIndex = "+nextFeatIndex+"\n"); + if (featDef.isByteFeature(nextFeatIndex)) { + // if we have a byte feature, build a byte decision node + numDaughters = featDef.getNumberOfValues(nextFeatIndex); + daughterNode = new DecisionNode.ByteDecisionNode(nextFeatIndex, numDaughters, featDef); + } else { + if (featDef.isShortFeature(nextFeatIndex)) { + // if we have a short feature, build a short decision node + numDaughters = featDef.getNumberOfValues(nextFeatIndex); + daughterNode = new DecisionNode.ShortDecisionNode(nextFeatIndex, numDaughters, featDef); + } else { + // feature is of type float, currently not supported in ffi + throw new IllegalArgumentException("Found float feature in FeatureFileIndexer!"); + } + } + + if (motherCARTNode == null) { + // if the mother is null, the current node is the root + rootNode = daughterNode; + daughterNode.setIsRoot(true); + } else { + // if the current node is not the root, + // set mother and daughter accordingly + motherCARTNode.addDaughter(daughterNode); + } + // for every daughter go in recursion + for (int i = 0; i < numDaughters; i++) { + MaryNode nextChild = currentTreeNode.getChild(i); + addDaughters(daughterNode, nextChild, ffi); + + } + } else { + // we are at a leaf node + // System.out.println("Adding leaf"); + // get the feature vectors + FeatureVector[] featureVectors = ffi.getFeatureVectors(currentTreeNode.getFrom(), currentTreeNode.getTo()); + // build a new leaf + LeafNode leaf = new LeafNode.FeatureVectorLeafNode(featureVectors); + + if (motherCARTNode == null) { + // if the mother is null, the current node is the root + rootNode = leaf; + } else { + // set mother and daughter + motherCARTNode.addDaughter(leaf); + } + } + } + +} diff --git a/marytts-unitselection/src/main/java/marytts/cart/LeafNode.java b/marytts-unitselection/src/main/java/marytts/cart/LeafNode.java new file mode 100644 index 0000000000..44614de7ab --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/cart/LeafNode.java @@ -0,0 +1,541 @@ +/** + * Copyright 2000-2009 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.cart; + +import java.util.ArrayList; +import java.util.List; + +import marytts.exceptions.MaryConfigurationException; +import marytts.features.FeatureDefinition; +import marytts.features.FeatureVector; + +/** + * The leaf of a CART. + */ +public abstract class LeafNode extends Node { + + public enum LeafType { + IntArrayLeafNode, FloatLeafNode, IntAndFloatArrayLeafNode, StringAndFloatLeafNode, FeatureVectorLeafNode, PdfLeafNode + }; + + // unique index used in MaryCART format + protected int uniqueLeafId; + + /** + * Create a new LeafNode. + * + * @param tok + * the String Tokenizer containing the String with the indices + * @param openBrackets + * the number of opening brackets at the first token + */ + public LeafNode() { + super(); + isRoot = false; + } + + @Override + public boolean isLeafNode() { + return true; + } + + /** + * Count all the nodes at and below this node. A leaf will return 1; the root node will report the total number of decision + * and leaf nodes in the tree. + * + * @return + */ + public int getNumberOfNodes() { + return 1; + } + + // unique index used in MaryCART format + public void setUniqueLeafId(int id) { + this.uniqueLeafId = id; + } + + public int getUniqueLeafId() { + return uniqueLeafId; + } + + public String toString() { + return "id" + uniqueLeafId; + } + + /** + * Indicate whether the leaf node has no meaningful data. + * + * @return + */ + public abstract boolean isEmpty(); + + /** + * Count all the data available at and below this node. The meaning of this depends on the type of nodes; for example, when + * IntArrayLeafNodes are used, it is the total number of ints that are saved in all leaf nodes below the current node. + * + * @return an int counting the data below the current node, or -1 if such a concept is not meaningful. + */ + public abstract int getNumberOfData(); + + /** + * Get all the data at or below this node. The type of data returned depends on the type of nodes; for example, when + * IntArrayLeafNodes are used, one int[] is returned which contains all int values in all leaf nodes below the current node. + * + * @return an object containing all data below the current node, or null if such a concept is not meaningful. + */ + public abstract Object getAllData(); + + /** + * Write this node's data into the target object at pos, making sure that exactly len data are written. The type of data + * written depends on the type of nodes; for example, when IntArrayLeafNodes are used, target would be an int[]. + * + * @param array + * the object to write to, usually an array. + * @param pos + * the position in the target at which to start writing + * @param len + * the amount of data items to write, usually equals getNumberOfData(). + */ + protected abstract void fillData(Object target, int pos, int len); + + /** + * The type of this leaf node. + * + * @return + */ + public abstract LeafType getLeafNodeType(); + + /** + * An LeafNode class suitable for representing the leaves of classification trees -- the leaf is a collection of items + * identified by an index number. + * + * @author marc + * + */ + public static class IntArrayLeafNode extends LeafNode { + protected int[] data; + + public IntArrayLeafNode(int[] data) { + super(); + this.data = data; + } + + /** + * Get all data in this leaf + * + * @return the int array contained in this leaf + */ + public Object getAllData() { + return data; + } + + public int[] getIntData() { + return data; + } + + protected void fillData(Object target, int pos, int len) { + if (!(target instanceof int[])) + throw new IllegalArgumentException("Expected target object of type int[], got " + target.getClass()); + int[] array = (int[]) target; + assert len <= data.length; + System.arraycopy(data, 0, array, pos, len); + } + + public int getNumberOfData() { + if (data != null) + return data.length; + return 0; + } + + public boolean isEmpty() { + return data == null || data.length == 0; + } + + public LeafType getLeafNodeType() { + return LeafType.IntArrayLeafNode; + } + + public String toString() { + if (data == null) + return super.toString() + "(int[null])"; + return super.toString() + "(int[" + data.length + "])"; + } + + } + + public static class IntAndFloatArrayLeafNode extends IntArrayLeafNode { + + protected float[] floats; + + public IntAndFloatArrayLeafNode(int[] data, float[] floats) { + super(data); + this.floats = floats; + } + + public float[] getFloatData() { + return floats; + } + + /** + * For the int-float pairs in this leaf, return the int value for which the associated float value is the highest one. If + * the float values are probabilities, this method returns the most probable int. + * + * @return + */ + public int mostProbableInt() { + int bestInd = 0; + float maxProb = 0f; + + for (int i = 0; i < data.length; i++) { + if (floats[i] > maxProb) { + maxProb = floats[i]; + bestInd = data[i]; + } + } + return bestInd; + } + + /** + * Delete a candidate of the leaf by its given data/index + * + * @param target + * the given data + */ + public void eraseData(int target) { + int[] newData = new int[data.length - 1]; + float[] newFloats = new float[floats.length - 1]; + int index = 0; + for (int i = 0; i < data.length; i++) { + if (data[i] != target) { + newData[index] = data[i]; + newFloats[index] = floats[i]; + index++; + } + } + data = newData; + floats = newFloats; + } + + public LeafType getLeafNodeType() { + return LeafType.IntAndFloatArrayLeafNode; + } + + public String toString() { + if (data == null) + return super.toString() + "(int and floats[null])"; + return super.toString() + "(int and floats[" + data.length + "])"; + } + + } + + public static class StringAndFloatLeafNode extends IntAndFloatArrayLeafNode { + + public StringAndFloatLeafNode(int[] data, float[] floats) { + super(data, floats); + } + + /** + * Return the most probable value in this leaf, translated into its string representation using the featureIndex'th + * feature of the given feature definition. + * + * @param featureDefinition + * @param featureIndex + * @return + */ + public String mostProbableString(FeatureDefinition featureDefinition, int featureIndex) { + int bestInd = mostProbableInt(); + return featureDefinition.getFeatureValueAsString(featureIndex, bestInd); + } + + public String toString() { + if (data == null) + return super.toString() + "(string and floats[null])"; + return super.toString() + "(string and floats[" + data.length + "])"; + } + + public LeafType getLeafNodeType() { + return LeafType.StringAndFloatLeafNode; + } + + } + + public static class FeatureVectorLeafNode extends LeafNode { + private FeatureVector[] featureVectors; + private List featureVectorList; + private boolean growable; + + /** + * Build a new leaf node containing the given feature vectors + * + * @param featureVectors + * the feature vectors + */ + public FeatureVectorLeafNode(FeatureVector[] featureVectors) { + super(); + this.featureVectors = featureVectors; + growable = false; + } + + /** + * Build a new, empty leaf node to be filled with vectors later + * + */ + public FeatureVectorLeafNode() { + super(); + featureVectorList = new ArrayList(); + featureVectors = null; + growable = true; + } + + public void addFeatureVector(FeatureVector fv) { + featureVectorList.add(fv); + } + + /** + * Get the feature vectors of this node + * + * @return the feature vectors + */ + public FeatureVector[] getFeatureVectors() { + if (growable && (featureVectors == null || featureVectors.length == 0)) { + featureVectors = (FeatureVector[]) featureVectorList.toArray(new FeatureVector[featureVectorList.size()]); + } + return featureVectors; + } + + public void setFeatureVectors(FeatureVector[] fv) { + this.featureVectors = fv; + } + + /** + * Get all data in this leaf + * + * @return the featurevector array contained in this leaf + */ + public Object getAllData() { + if (growable && (featureVectors == null || featureVectors.length == 0)) { + featureVectors = (FeatureVector[]) featureVectorList.toArray(new FeatureVector[featureVectorList.size()]); + } + return featureVectors; + } + + protected void fillData(Object target, int pos, int len) { + if (!(target instanceof FeatureVector[])) + throw new IllegalArgumentException("Expected target object of type FeatureVector[], got " + target.getClass()); + FeatureVector[] array = (FeatureVector[]) target; + assert len <= featureVectors.length; + System.arraycopy(featureVectors, 0, array, pos, len); + } + + public int getNumberOfData() { + if (growable) { + return featureVectorList.size(); + } + if (featureVectors != null) + return featureVectors.length; + return 0; + } + + public boolean isEmpty() { + return featureVectors == null || featureVectors.length == 0; + } + + public LeafType getLeafNodeType() { + return LeafType.FeatureVectorLeafNode; + } + + public String toString() { + if (growable) + return "fv[" + featureVectorList.size() + "]"; + if (featureVectors == null) + return super.toString() + "(fv[null])"; + return super.toString() + "(fv[" + featureVectors.length + "])"; + } + + } + + /** + * A leaf class that is suitable for regression trees. Here, a leaf consists of a mean and a standard deviation. + * + * @author marc + * + */ + public static class FloatLeafNode extends LeafNode { + private float[] data; + + public FloatLeafNode(float[] data) { + super(); + if (data.length != 2) + throw new IllegalArgumentException("data must have length 2, found " + data.length); + this.data = data; + } + + /** + * Get all data in this leaf + * + * @return the mean/standard deviation value contained in this leaf + */ + public Object getAllData() { + return data; + } + + public int getDataLength() { + return data.length; + } + + public float getMean() { + return data[1]; + } + + public float getStDeviation() { + return data[0]; + } + + protected void fillData(Object target, int pos, int len) { + throw new IllegalStateException("This method should not be called for FloatLeafNodes"); + } + + public int getNumberOfData() { + return 1; + } + + public boolean isEmpty() { + return false; + } + + public LeafType getLeafNodeType() { + return LeafType.FloatLeafNode; + } + + public String toString() { + if (data == null) + return super.toString() + "(mean=null, stddev=null)"; + return super.toString() + "(mean=" + data[1] + ", stddev=" + data[0] + ")"; + } + + } + + /** + * A leaf class that is suitable for regression trees. Here, a leaf consists of a mean and a diagonal covariance vectors. This + * node will be used in HTS CART trees. + * + */ + public static class PdfLeafNode extends LeafNode { + private int vectorSize; + private double[] mean; // mean vector. + private double[] variance; // diagonal covariance. + private double voicedWeight; // only for lf0 tree. + + /** + * @param idx + * , a unique index number + * @param pdf + * , pdf[numStreams][2*vectorSize] + */ + public PdfLeafNode(int idx, double pdf[][]) throws MaryConfigurationException { + super(); + this.setUniqueLeafId(idx); + // System.out.println("adding leaf node: " + idx); + if (pdf != null) { + double val; + int i, j, vsize, nstream; + nstream = pdf.length; + + if (nstream == 1) { // This is the case for dur, mgc, str, mag, or joinModel. + vsize = (pdf[0].length) / 2; + vectorSize = vsize; + mean = new double[vsize]; + for (i = 0, j = 0; j < vsize; i++, j++) + mean[i] = pdf[0][j]; + variance = new double[vsize]; + for (i = 0, j = vsize; j < (2 * vsize); i++, j++) + variance[i] = pdf[0][j]; + + } else { // this is the case for lf0 + vectorSize = nstream; + mean = new double[nstream]; + variance = new double[nstream]; + for (int stream = 0; stream < nstream; stream++) { + mean[stream] = pdf[stream][0]; + variance[stream] = pdf[stream][1]; + // vw = lf0pdf[numStates][numPdfs][numStreams][2]; /* voiced weight */ + // uvw = lf0pdf[numStates][numPdfs][numStreams][3]; /* unvoiced weight */ + if (stream == 0) + voicedWeight = pdf[stream][2]; + } + } + } else { + throw new MaryConfigurationException("PdfLeafNode: pdf vector is null for index=" + idx); + } + + } + + public int getDataLength() { + return mean.length; + } + + public double[] getMean() { + return mean; + } + + public double[] getVariance() { + return variance; + } + + public double getVoicedWeight() { + return voicedWeight; + } + + public int getVectorSize() { + return vectorSize; + } + + protected void fillData(Object target, int pos, int len) { + throw new IllegalStateException("This method should not be called for PdfLeafNodes"); + } + + // not meaningful here. + public Object getAllData() { + return null; + } + + // not meaningful here. + // i need this value positive when searching ??? + public int getNumberOfData() { + return 1; + } + + public boolean isEmpty() { + return false; + } + + public LeafType getLeafNodeType() { + return LeafType.PdfLeafNode; + } + + public String toString() { + if (mean == null) + return super.toString() + "(mean=null, stddev=null)"; + return super.toString() + "(mean=[" + vectorSize + "], stddev=[" + vectorSize + "])"; + } + + } + +} diff --git a/marytts-unitselection/src/main/java/marytts/cart/Node.java b/marytts-unitselection/src/main/java/marytts/cart/Node.java new file mode 100644 index 0000000000..4ebf91f571 --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/cart/Node.java @@ -0,0 +1,172 @@ +/** + * Portions Copyright 2006 DFKI GmbH. + * Portions Copyright 2001 Sun Microsystems, Inc. + * Portions Copyright 1999-2001 Language Technologies Institute, + * Carnegie Mellon University. + * All Rights Reserved. Use is subject to license terms. + * + * Permission is hereby granted, free of charge, to use and distribute + * this software and its documentation without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of this work, and to + * permit persons to whom this work is furnished to do so, subject to + * the following conditions: + * + * 1. The code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * 2. Any modifications must be clearly marked as such. + * 3. Original authors' names are not deleted. + * 4. The authors' names are not used to endorse or promote products + * derived from this software without specific prior written + * permission. + * + * DFKI GMBH AND THE CONTRIBUTORS TO THIS WORK DISCLAIM ALL WARRANTIES WITH + * REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL DFKI GMBH NOR THE + * CONTRIBUTORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL + * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR + * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS + * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF + * THIS SOFTWARE. + */ +package marytts.cart; + +/** + * A node for the CART or DirectedGraph. All node types inherit from this class + */ +public abstract class Node { + // isRoot should be set to true if this node is the root node + protected boolean isRoot; + + // every node except the root node has a mother + protected Node mother; + + // the index of the node in the daughters array of its mother + protected int nodeIndex; + + /** + * set the mother node of this node, and remember this node's index in mother. + * + * @param node + * the mother node + * @param nodeIndex + * the index of this node in the mother node's list of daughters + */ + public void setMother(Node node, int nodeIndex) { + this.mother = node; + this.nodeIndex = nodeIndex; + } + + /** + * Get the mother node of this node + * + * @return the mother node + */ + public Node getMother() { + return mother; + } + + /** + * Get the index of this node in the mother's array of daughters + * + * @return the index + */ + public int getNodeIndex() { + return nodeIndex; + } + + /** + * Set isRoot to the given value + * + * @param isRoot + * the new value of isRoot + */ + public void setIsRoot(boolean isRoot) { + this.isRoot = isRoot; + } + + /** + * Get the setting of isRoot + * + * @return the setting of isRoot + */ + public boolean isRoot() { + return isRoot; + } + + public boolean isDecisionNode() { + return false; + } + + public boolean isLeafNode() { + return false; + } + + public boolean isDirectedGraphNode() { + return false; + } + + public Node getRootNode() { + if (isRoot) { + assert mother == null; + return this; + } else { + assert mother != null : " I am not root but I have no mother :-("; + return mother.getRootNode(); + } + } + + public String getDecisionPath() { + String ancestorInfo; + if (mother == null) + ancestorInfo = "null"; + else if (mother.isDecisionNode()) { + ancestorInfo = ((DecisionNode) mother).getDecisionPath(getNodeIndex()); + } else { + ancestorInfo = mother.getDecisionPath(); + } + return ancestorInfo + " - " + toString(); + } + + /** + * Count all the nodes at and below this node. A leaf will return 1; the root node will report the total number of decision + * and leaf nodes in the tree. + * + * @return + */ + public abstract int getNumberOfNodes(); + + /** + * Count all the data available at and below this node. The meaning of this depends on the type of nodes; for example, when + * IntArrayLeafNodes are used, it is the total number of ints that are saved in all leaf nodes below the current node. + * + * @return an int counting the data below the current node, or -1 if such a concept is not meaningful. + */ + public abstract int getNumberOfData(); + + /** + * Get all the data at or below this node. The type of data returned depends on the type of nodes; for example, when + * IntArrayLeafNodes are used, one int[] is returned which contains all int values in all leaf nodes below the current node. + * + * @return an object containing all data below the current node, or null if such a concept is not meaningful. + */ + public abstract Object getAllData(); + + /** + * Write this node's data into the target object at pos, making sure that exactly len data are written. The type of data + * written depends on the type of nodes; for example, when IntArrayLeafNodes are used, target would be an int[]. + * + * @param array + * the object to write to, usually an array. + * @param pos + * the position in the target at which to start writing + * @param len + * the amount of data items to write, usually equals getNumberOfData(). + */ + protected abstract void fillData(Object target, int pos, int len); + + public String toString(String prefix) { + return prefix + this.toString(); + } + +} \ No newline at end of file diff --git a/marytts-unitselection/src/main/java/marytts/cart/NodeIterator.java b/marytts-unitselection/src/main/java/marytts/cart/NodeIterator.java new file mode 100644 index 0000000000..f3a462f7ca --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/cart/NodeIterator.java @@ -0,0 +1,163 @@ +/** + * Copyright 2009 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ + +package marytts.cart; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; + +/** + * @author marc + * + */ +public class NodeIterator implements Iterator { + private Node root; + private Node current; + private boolean showLeafNodes; + private boolean showDecisionNodes; + private boolean showDirectedGraphNodes; + private Set alreadySeen = new HashSet(); + // we need to keep our own map of daughter-mother relationships, + // because for subgraphs, we could move out of a subgraph if we call node.getMother() + // if the mother via which we entered a multi-parent node is not the first mother. + private Map daughterToMother = new HashMap(); + + /** + * Iterate over all nodes in the graph. + * + * @param graph + * @param showLeafNodes + * @param showDecisionNodes + * @param showDirectedGraphNodes + */ + protected NodeIterator(DirectedGraph graph, boolean showLeafNodes, boolean showDecisionNodes, boolean showDirectedGraphNodes) { + this(graph.getRootNode(), showLeafNodes, showDecisionNodes, showDirectedGraphNodes); + } + + /** + * Iterate over the subtree below rootNode. + * + * @param rootNode + * @param showLeafNodes + * @param showDecisionNodes + * @param showDirectedGraphNodes + */ + protected NodeIterator(Node rootNode, boolean showLeafNodes, boolean showDecisionNodes, boolean showDirectedGraphNodes) { + this.root = rootNode; + this.showLeafNodes = showLeafNodes; + this.showDecisionNodes = showDecisionNodes; + this.showDirectedGraphNodes = showDirectedGraphNodes; + this.current = root; + alreadySeen.add(current); + if (!currentIsSuitable()) { + nextSuitableNodeDepthFirst(); + } + } + + public boolean hasNext() { + return current != null; + } + + public T next() { + T ret = (T) current; + // and already prepare the current one + nextSuitableNodeDepthFirst(); + return ret; + } + + private boolean currentIsSuitable() { + return (current == null || showDecisionNodes && current.isDecisionNode() || showLeafNodes && current.isLeafNode() || showDirectedGraphNodes + && current.isDirectedGraphNode()); + } + + private void nextSuitableNodeDepthFirst() { + do { + nextNodeDepthFirst(); + } while (!currentIsSuitable()); + } + + private void nextNodeDepthFirst() { + if (current == null) + return; + if (current.isDecisionNode()) { + DecisionNode dec = (DecisionNode) current; + for (int i = 0; i < dec.getNumberOfDaugthers(); i++) { + Node daughter = dec.getDaughter(i); + if (daughter == null) + continue; + daughterToMother.put(daughter, dec); + if (unseenNode(dec.getDaughter(i))) + return; + } + } else if (current.isDirectedGraphNode()) { + // Graph nodes return leaf child first, then decision child + DirectedGraphNode g = (DirectedGraphNode) current; + Node leaf = g.getLeafNode(); + if (leaf != null) { + daughterToMother.put(leaf, g); + if (unseenNode(leaf)) + return; + } + Node dec = g.getDecisionNode(); + if (dec != null) { + daughterToMother.put(dec, g); + if (unseenNode(dec)) + return; + } + } + // If we didn't find a suitable child, we need to: + backtrace(); + } + + private void backtrace() { + // Only go back to mothers we have come from. + // This has two effects: + // 1. We cannot go beyond root node; + // 2. we don't risk to leave the subgraph defined by root node + // in cases where we enter into a multi-parent node from a not-first mother + // (in such cases, getMother() would return the first mother). + current = daughterToMother.get(current); + nextNodeDepthFirst(); + } + + /** + * Test whether the given node is unseen. If so, move current to it, and remember it as a seen node. + * + * @param candidate + * @return + */ + private boolean unseenNode(Node candidate) { + if (candidate != null && !alreadySeen.contains(candidate)) { + current = candidate; + alreadySeen.add(current); + return true; + } + return false; + + } + + public void remove() { + throw new UnsupportedOperationException("Cannot remove nodes using this iterator"); + } + +} diff --git a/marytts-unitselection/src/main/java/marytts/cart/StringPredictionTree.java b/marytts-unitselection/src/main/java/marytts/cart/StringPredictionTree.java new file mode 100644 index 0000000000..8f9c7c902a --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/cart/StringPredictionTree.java @@ -0,0 +1,222 @@ +/** + * Copyright 2000-2009 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.cart; + +import java.io.BufferedReader; +import java.io.IOException; +import java.util.regex.Pattern; + +import marytts.cart.LeafNode.IntAndFloatArrayLeafNode; +import marytts.cart.LeafNode.LeafType; +import marytts.cart.io.WagonCARTReader; +import marytts.features.FeatureDefinition; +import marytts.features.FeatureVector; +import marytts.features.Target; + +//import com.sun.tools.javac.code.Attribute.Array; + +//public class StringPredictionTree extends ExtendedClassificationTree { +public class StringPredictionTree extends CART { + public static final String ENC_LINE_START = ";;target={"; + public static final String ENC_LINE_END = "}\n"; + + // TODO: maybe use an HashMap + // this strores the strings that correspond to the indices at the leaves + String[] stringIdDecoding; + Pattern splitPattern = Pattern.compile("'"); + Pattern delimPattern = Pattern.compile(",\\d+:|}$"); + + /** + * + * @param rootNode + * the root node of this tree. This node has to be set to be a root node beforehand. + * @param featDef + * the featureDefinition used in this tree + * + * @author ben + */ + public StringPredictionTree(Node aRootNode, FeatureDefinition aFeatDef, String[] aTargetDecoding) { + if (!aRootNode.isRoot()) + throw new IllegalArgumentException("Tried to set a non-root-node as root of the tree. "); + + this.rootNode = aRootNode; + this.featDef = aFeatDef; + this.stringIdDecoding = aTargetDecoding; + } + + /** + * + * This constructs a new string prediciton tree from a stream containing a tree in wagon format. In addition to the + * constructor of ExtendedClassificationTree it reads in the mapping from numbers to the Strings from a stream. The encoding + * has to be the first line in the file (a empty line is allowed). + * + * It has the form: + * + * ;;target={1:'string_a',2:'string_b,'...',26:'string_z'} + * + */ + + public StringPredictionTree(BufferedReader reader, FeatureDefinition featDefinition) throws IOException { + + String line = reader.readLine(); + + if (line.equals("")) {// first line is empty, read again + line = reader.readLine(); + } + + if (line.startsWith(ENC_LINE_START)) { + + // split of the beginning of the string + String rawLine = line.substring((ENC_LINE_START + "0:'").length()); + + // regular expression for splitting of the target encodings + // ',NUMBER:' OR '} + + String[] splitted = splitPattern.split(rawLine); + + this.stringIdDecoding = new String[splitted.length / 2]; + + for (int i = 0; i < splitted.length / 2; i++) { + this.stringIdDecoding[i] = splitted[i * 2]; + if (!this.delimPattern.matcher(splitted[i * 2 + 1]).matches()) { + throw new IllegalArgumentException("wrong encoding for the mapping of numbers and strings."); + } + } + + // System.err.println(rawLine); + // System.err.println(Arrays.toString(stringIdDecoding)); + + // encoding/linebreak problems with this line? + // this.stringIdDecoding = rawLine.split("',\\d+:'|'}$"); + + } else + throw new IllegalArgumentException("First line must be a comment line specifying the target symbols."); + + // read the rest of the tree + // old: this.load(reader, featDefinition); + // CHECK!! this has not been tested, maybe it does not work!!! + WagonCARTReader wagonReader = new WagonCARTReader(LeafType.IntAndFloatArrayLeafNode); + this.setRootNode(wagonReader.load(reader, featDefinition)); + + } + + // toString method, that writes the decoding in first line, + // should be something like: + // ;;target={1:'string_a',2:'string_b',...,26:'string_z'} + // this is followed by a + public String toString() { + + // make String representation of target symbol decoding and invoke super-toString + StringBuilder sb = new StringBuilder(); + + sb.append(ENC_LINE_START); + + for (int i = 0; i < this.stringIdDecoding.length; i++) { + + if (i > 0) + sb.append(","); + + sb.append(i); + sb.append(":'"); + sb.append(this.stringIdDecoding[i]); + sb.append("'"); + } + + sb.append(ENC_LINE_END); + sb.append(super.toString()); + + return sb.toString(); + } + + /** + * TODO: copied from CART, does not work as expected with minNumberOfData = 0 + * + * Passes the given item through this CART and returns the leaf Node, or the Node it stopped walking down. + * + * @param target + * the target to analyze + * @param minNumberOfData + * the minimum number of data requested. If this is 0, walk down the CART until the leaf level. + * + * @return the Node + */ + public Node interpretToNode(FeatureVector featureVector, int minNumberOfData) { + Node currentNode = rootNode; + Node prevNode = null; + + // logger.debug("Starting cart at "+nodeIndex); + while (currentNode.getNumberOfData() > minNumberOfData && !(currentNode instanceof LeafNode)) { + // while we have not reached the bottom, + // get the next node based on the features of the target + prevNode = currentNode; + currentNode = ((DecisionNode) currentNode).getNextNode(featureVector); + // logger.debug(decision.toString() + " result '"+ + // decision.findFeature(item) + "' => "+ nodeIndex); + } + + // Now usually we will have gone down one level too far + if (currentNode.getNumberOfData() < minNumberOfData && prevNode != null) { + currentNode = prevNode; + } + + assert currentNode.getNumberOfData() >= minNumberOfData || currentNode == rootNode; + + return currentNode; + + } + + public String getMostProbableString(FeatureVector aFV) { + + // get the node data + // TODO: for some reason, when I changed interpretToNode in taking a fv, I had to change mindata to -1 ?! + IntAndFloatArrayLeafNode predictedNode = (IntAndFloatArrayLeafNode) this.interpretToNode(aFV, -1); + + // look for the index with highest associated probability + float[] probs = predictedNode.getFloatData(); + int[] indices = predictedNode.getIntData(); + + int bestInd = 0; + float maxProb = 0f; + + for (int i = 0; i < indices.length; i++) { + if (probs[i] > maxProb) { + maxProb = probs[i]; + bestInd = indices[i]; + } + } + + if (bestInd >= stringIdDecoding.length) { + logger.info("looking up most probable string for feature vector"); + logger.error("index bigger than number of targets"); + logger.info("biggest index is " + (stringIdDecoding.length - 1) + "with the symbol" + + stringIdDecoding[stringIdDecoding.length - 1]); + } + + // get the String representation + return this.stringIdDecoding[bestInd]; + } + + public String getMostProbableString(Target aTarget) { + // get the String representation + return this.getMostProbableString(aTarget.getFeatureVector()); + + } + +} diff --git a/marytts-unitselection/src/main/java/marytts/cart/impose/FeatureArrayIndexer.java b/marytts-unitselection/src/main/java/marytts/cart/impose/FeatureArrayIndexer.java new file mode 100644 index 0000000000..d6a693d064 --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/cart/impose/FeatureArrayIndexer.java @@ -0,0 +1,422 @@ +/** + * Copyright 2006 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.cart.impose; + +import java.util.Arrays; + +import marytts.features.FeatureDefinition; +import marytts.features.FeatureVector; + +/** + * A class branched from FeatureFileIndexer which works directly on a feature array, rather than extending FeatureFileReader. + * + * @author Marc Schröder + * + */ +public class FeatureArrayIndexer { + + private MaryNode tree = null; + private int[] featureSequence = null; + private FeatureComparator c = new FeatureComparator(-1, null); + private UnitIndexComparator cui = new UnitIndexComparator(); + private FeatureVector[] featureVectors; + private FeatureDefinition featureDefinition; + + private long numberOfLeaves = 0; + + /****************/ + /* CONSTRUCTORS */ + /****************/ + + /** + * Constructor which takes an array of feature vectors and launches an indexing operation according to a feature sequence + * constraint. + * + * @param featureVectors + * an array of feature vectors + * @param featureDefinition + * a feature definition to make sense of the feature vectors + * @param setFeatureSequence + * An array of indexes indicating the hierarchical order (or, equivalently, the sequence) of the features to use + * for the indexing. + * + */ + public FeatureArrayIndexer(FeatureVector[] featureVectors, FeatureDefinition featureDefinition, int[] setFeatureSequence) { + this(featureVectors, featureDefinition); + deepSort(setFeatureSequence); + } + + /** + * Constructor which takes an array of feature vectors and launches an indexing operation according to a feature sequence + * constraint. + * + * @param featureVectors + * an array of feature vectors + * @param featureDefinition + * a feature definition to make sense of the feature vectors + * @param setFeatureSequence + * An array of feature names indicating the hierarchical order (or, equivalently, the sequence) of the features to + * use for the indexing. + * + */ + public FeatureArrayIndexer(FeatureVector[] featureVectors, FeatureDefinition featureDefinition, String[] setFeatureSequence) { + this(featureVectors, featureDefinition); + deepSort(setFeatureSequence); + } + + /** + * Constructor which loads the feature vector array but does not launch an indexing operation. + * + * @param featureVectors + * an array of feature vectors + * @param featureDefinition + * a feature definition to make sense of the feature vectors + * + */ + public FeatureArrayIndexer(FeatureVector[] featureVectors, FeatureDefinition featureDefinition) { + this.featureVectors = featureVectors; + this.featureDefinition = featureDefinition; + } + + /********************/ + /* INDEXING METHODS */ + /********************/ + + /** + * A local sort at a particular node along the deep sorting operation. This is a recursive function. + * + * @param currentFeatureIdx + * The currently tested feature. + * @param currentNode + * The current node, holding the currently processed zone in the array of feature vectors. + */ + private void sortNode(int currentFeatureIdx, MaryNode currentNode) { + /* If we have reached a leaf, do a final sort according to the unit index and return: */ + if (currentFeatureIdx == featureSequence.length) { + Arrays.sort(featureVectors, currentNode.from, currentNode.to, cui); + numberOfLeaves++; + /* + * System.out.print( "LEAF ! (" + (currentNode.to-currentNode.from) + " units)" ); for ( int i = currentNode.from; i < + * currentNode.to; i++ ) { System.out.print( " (" + featureVectors[i].getUnitIndex() + " 0)" ); } System.out.println( + * "" ); + */ + return; + } + /* Else: */ + int currentFeature = featureSequence[currentFeatureIdx]; + FeatureVector.FeatureType featureType = featureVectors[0].getFeatureType(currentFeature); + /* Register the feature currently used for the splitting */ + currentNode.setFeatureIndex(currentFeature); + /* Perform the sorting according to the currently considered feature: */ + /* 1) position the comparator onto the right feature */ + c.setFeatureIdx(currentFeature, featureType); + /* 2) do the sorting */ + Arrays.sort(featureVectors, currentNode.from, currentNode.to, c); + + /* + * Then, seek for the zones where the feature value is the same, and launch the next sort level on these. + */ + int nVal = featureDefinition.getNumberOfValues(currentFeature); + currentNode.split(nVal); + int nextFrom = currentNode.from; + int nextTo = currentNode.from; + for (int i = 0; i < nVal; i++) { + nextFrom = nextTo; + // System.out.print( "Next node begins at " + nextFrom ); + while ((nextTo < currentNode.to) && (featureVectors[nextTo].getFeatureAsInt(currentFeature) == i)) { + // System.out.print( " " + featureVectors[nextTo].getFeatureAsInt( currentFeature ) ); + nextTo++; + } + // System.out.println( " and ends at " + nextTo + " for a total of " + (nextTo-nextFrom) + " units." ); + if ((nextTo - nextFrom) != 0) { + MaryNode nod = new MaryNode(nextFrom, nextTo); + currentNode.setChild(i, nod); + // System.out.print("(" + i + " isByteOf " + currentFeature + ")" ); + sortNode(currentFeatureIdx + 1, nod); + } else + currentNode.setChild(i, null); + } + } + + /** + * Launches a deep sort on the array of feature vectors. This is public because it can be used to re-index the previously read + * feature file. + * + * @param featureIdx + * An array of feature indexes, indicating the sequence of features according to which the sorting should be + * performed. + */ + public void deepSort(int[] setFeatureSequence) { + featureSequence = setFeatureSequence; + numberOfLeaves = 0; + tree = new MaryNode(0, featureVectors.length); + sortNode(0, tree); + } + + /** + * Launches a deep sort on the array of feature vectors. This is public because it can be used to re-index the previously read + * feature file. + * + * @param featureIdx + * An array of feature names, indicating the sequence of features according to which the sorting should be + * performed. + */ + public void deepSort(String[] setFeatureSequence) { + featureSequence = featureDefinition.getFeatureIndexArray(setFeatureSequence); + numberOfLeaves = 0; + tree = new MaryNode(0, featureVectors.length); + sortNode(0, tree); + } + + /** + * Fill a particular node of a pre-specified tree. This is a recursive function. + * + * @param currentNode + * The current node, holding the currently processed zone in the array of feature vectors. + */ + private void fillNode(MaryNode currentNode) { + /* If we have reached a leaf, do a final sort according to the unit index and return: */ + if (currentNode.isLeaf()) { + Arrays.sort(featureVectors, currentNode.from, currentNode.to, cui); + numberOfLeaves++; + /* + * System.out.print( "LEAF ! (" + (currentNode.to-currentNode.from) + " units)" ); for ( int i = currentNode.from; i < + * currentNode.to; i++ ) { System.out.print( " (" + featureVectors[i].getUnitIndex() + " 0)" ); } System.out.println( + * "" ); + */ + return; + } + /* Else: */ + int currentFeature = currentNode.featureIndex; + FeatureVector.FeatureType featureType = featureVectors[0].getFeatureType(currentFeature); + /* Perform the sorting according to the currently considered feature: */ + /* 1) position the comparator onto the right feature */ + c.setFeatureIdx(currentFeature, featureType); + /* 2) do the sorting */ + Arrays.sort(featureVectors, currentNode.from, currentNode.to, c); + + /* + * Then, seek for the zones where the feature value is the same, and launch the next sort level on these. + */ + int nVal = featureDefinition.getNumberOfValues(currentFeature); + int nextFrom = currentNode.from; + int nextTo = currentNode.from; + for (int i = 0; i < nVal; i++) { + nextFrom = nextTo; + // System.out.print( "Next node begins at " + nextFrom ); + while ((nextTo < currentNode.to) && (featureVectors[nextTo].getFeatureAsInt(currentFeature) == i)) { + // System.out.print( " " + featureVectors[nextTo].getFeatureAsInt( currentFeature ) ); + nextTo++; + } + // System.out.println( " and ends at " + nextTo + " for a total of " + (nextTo-nextFrom) + " units." ); + if ((nextTo - nextFrom) != 0) { + MaryNode nod = currentNode.getChild(i); + if (nod != null) { + nod.from = nextFrom; + nod.to = nextTo; + fillNode(nod); + } + } else + currentNode.setChild(i, null); + } + } + + /** + * Fill a tree which specifies a feature hierarchy but no corresponding units. + * + * @param featureIdx + * An array of feature indexes, indicating the sequence of features according to which the sorting should be + * performed. + */ + public void deepFill(MaryNode specTree) { + tree = specTree; + numberOfLeaves = 0; + sortNode(0, tree); + } + + /***************************/ + /* QUERY/RETRIEVAL METHODS */ + /***************************/ + + /** + * Retrieve an array of unit features which complies with a specific target specification, according to an underlying tree. + * + * @param v + * A feature vector for which to send back an array of complying unit indexes. + * @return A query result, comprising an array of feature vectors and the depth level which was actually reached. + * + * @see FeatureArrayIndexer#deepSort(int[]) + * @see FeatureArrayIndexer#deepFill(MaryNode) + */ + public FeatureFileIndexingResult retrieve(FeatureVector v) { + int level = 0; + /* Check if the tree is there */ + if (tree == null) { + throw new RuntimeException("Can't retrieve candidate units if a tree has not been built." + + " (Run this.deepSort(int[]) or this.deepFill(MaryNode) first.)"); + } + /* Walk down the tree */ + MaryNode n = tree; + MaryNode next = null; + while (!n.isLeaf()) { + next = n.getChild(v.getFeatureAsInt(n.getFeatureIndex())); + /* Check if the next node is a dead branch */ + if (next != null) { + n = next; + level++; + } else + break; + } + /* Dereference the reached node or leaf */ + FeatureFileIndexingResult qr = new FeatureFileIndexingResult(getFeatureVectors(n.from, n.to), level); + return (qr); + } + + /** + * Retrieve an array of unit features which complies with a specific target specification, according to an underlying tree, + * and given a stopping condition. + * + * @param v + * A feature vector for which to send back an array of complying unit indexes. + * @param condition + * A constant indicating a stopping criterion, among: FeatureFileIndexer.MAXDEPTH : walk the tree until its leaves + * (maximum depth); FeatureFileIndexer.MAXLEVEL : walk the tree until a certain depth level; + * FeatureFileIndexer.MINUNITS : walk the tree until a certain number of units is reached. + * @param parameter + * A parameter interpreted according to the above condition: MAXDEPTH -> parameter is ignored; MAXLEVEL -> + * parameter = maximum level to reach; MINUNITS -> parameter = lower bound on the number of units to return. + * + * @return A query result, comprising an array of feature vectors and the depth level which was actually reached. + * + * @see FeatureArrayIndexer#deepSort(int[]) + * @see FeatureArrayIndexer#deepFill(MaryNode) + */ + public static final int MAXDEPTH = 0; + public static final int MAXLEVEL = 1; + public static final int MINUNITS = 2; + + public FeatureFileIndexingResult retrieve(FeatureVector v, int condition, int parameter) { + int level = 0; + /* Check if the tree is there */ + if (tree == null) { + throw new RuntimeException("Can't retrieve candidate units if a tree has not been built." + + " (Run this.deepSort(int[]) or this.deepFill(MaryNode) first.)"); + } + // /**/ + // /* TODO: Do we want the warning below? */ + // /*if ( (condition == MAXLEVEL) && (featureSequence != null) && (parameter > featureSequence.length) ) { + // System.out.println( "WARNING: you asked for more levels [" + maxLevel + // + "] than the length of the underlying feature sequence[" + featureSequence.length + "]. Proceeding anyways." ); + // }*/ + /* Walk down the tree */ + MaryNode n = tree; + MaryNode next = null; + while (!n.isLeaf()) { + next = n.getChild(v.getFeatureAsInt(n.getFeatureIndex())); + /* Check for the number of units in the next node */ + if ((condition == MINUNITS) && ((next.to - next.from) < parameter)) + break; + /* Check if the next node is a dead branch */ + if (next != null) { + n = next; + level++; + } else + break; + /* Check for the current level */ + if ((condition == MAXLEVEL) && (level == parameter)) + break; + } + /* Dereference the reached node or leaf */ + FeatureFileIndexingResult qr = new FeatureFileIndexingResult(getFeatureVectors(n.from, n.to), level); + return (qr); + } + + /***************************/ + /* MISCELLANEOUS ACCESSORS */ + /***************************/ + + /** + * Get the feature sequence, as an information about the underlying tree structure. + * + * @return the feature sequence + */ + public int[] getFeatureSequence() { + return featureSequence; + } + + /** + * Get the tree + * + * @return the tree + */ + public MaryNode getTree() { + return tree; + } + + /** + * Get the feature vectors from the big array according to the given indices + * + * @param from + * the start index + * @param to + * the end index + * @return the feature vectors + */ + public FeatureVector[] getFeatureVectors(int from, int to) { + FeatureVector[] vectors = new FeatureVector[to - from]; + for (int i = from; i < to; i++) { + vectors[i - from] = featureVectors[i]; + } + return vectors; + } + + public FeatureDefinition getFeatureDefinition() { + return featureDefinition; + } + + /** + * Get the number of leaves. + * + * @return The number of leaves, or -1 if the tree has not been computed. + */ + public long getNumberOfLeaves() { + if (tree == null) + return (-1); + return (numberOfLeaves); + } + + /** + * Get the theoretical number of leaves, given a feature sequence. + * + * @return The number of leaves, or -1 if the capacity of the long integer was blown. + */ + public long getTheoreticalNumberOfLeaves(int[] feaSeq) { + long ret = 1; + for (int i = 0; i < feaSeq.length; i++) { + // System.out.println( "Feature [" + i + "] has [" + featureDefinition.getNumberOfValues( featureSequence[i] ) + + // "] values." + // + "(Number of leaves = [" + ret + "].)" ); + ret *= featureDefinition.getNumberOfValues(feaSeq[i]); + if (ret < 0) + return (-1); + } + return (ret); + } +} diff --git a/marytts-unitselection/src/main/java/marytts/cart/impose/FeatureComparator.java b/marytts-unitselection/src/main/java/marytts/cart/impose/FeatureComparator.java new file mode 100644 index 0000000000..d1b14967f1 --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/cart/impose/FeatureComparator.java @@ -0,0 +1,118 @@ +/** + * Copyright 2006 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.cart.impose; + +import java.util.Comparator; + +import marytts.features.FeatureVector; + +public class FeatureComparator implements Comparator { + + /** The index of the feature to be compared in the feature vector. */ + private int I = -1; + private FeatureVector.FeatureType type = null; + + /** + * Constructor which initializes the feature index. + * + * @param setI + * The index of the feature to be compared on the next run of the comparator. + */ + public FeatureComparator(int setI, FeatureVector.FeatureType featureType) { + setFeatureIdx(setI, featureType); + } + + /** + * Accessor to set the feature index. + * + * @param setI + * The index of the feature to be compared on the next run of the comparator. + */ + public void setFeatureIdx(int setI, FeatureVector.FeatureType featureType) { + I = setI; + type = featureType; + } + + /** + * Access the index of the currently compared feature. + * + * @return The index of the feature which the comparator currently deals with. + */ + public int getFeatureIdx() { + return (I); + } + + /** + * Compares two feature vectors according to their values at an internal index previously set by this.setFeatureIdx(). + * + * @param v1 + * The first vector. + * @param v2 + * The second vector. + * @return a negative integer, zero, or a positive integer as the feature at index I for v1 is less than, equal to, or greater + * than the feature at index I for v2. + * + * @see FeatureComparator#setFeatureIdx(int) + */ + public int compare(FeatureVector a, FeatureVector b) { + switch (type) { + case byteValued: + return a.byteValuedDiscreteFeatures[I] - b.byteValuedDiscreteFeatures[I]; + case shortValued: + int offset = a.byteValuedDiscreteFeatures.length; + return a.shortValuedDiscreteFeatures[I - offset] - b.shortValuedDiscreteFeatures[I - offset]; + case floatValued: + int offset2 = a.byteValuedDiscreteFeatures.length + a.shortValuedDiscreteFeatures.length; + float delta = a.continuousFeatures[I - offset2] - b.continuousFeatures[I - offset2]; + if (delta > 0) + return 1; + else if (delta < 0) + return -1; + return 0; + default: + throw new IllegalStateException("compare called with feature index " + I + " and feature type " + type); + } + + } + + /** + * The equals() method asked for by the Comparable interface. Returns true if the compared object is a FeatureComparator with + * the same internal index, false otherwise. + */ + public boolean equals(Object obj) { + if (!(obj instanceof FeatureComparator)) + return false; + else if (((FeatureComparator) obj).getFeatureIdx() != this.I) + return false; + return (true); + } +} + +/** + * An additional comparator for the unit indexes in the feature vectors. + * + */ +class UnitIndexComparator implements Comparator { + + public int compare(FeatureVector a, FeatureVector b) { + return a.unitIndex - b.unitIndex; + } + +} diff --git a/marytts-unitselection/src/main/java/marytts/cart/impose/FeatureFileIndexingResult.java b/marytts-unitselection/src/main/java/marytts/cart/impose/FeatureFileIndexingResult.java new file mode 100644 index 0000000000..bf9e01b157 --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/cart/impose/FeatureFileIndexingResult.java @@ -0,0 +1,46 @@ +/** + * Copyright 2006 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.cart.impose; + +import marytts.features.FeatureVector; + +/** + * A helper class to return the query results from the FeatureFileIndexer. + * + * @author sacha + * + */ +public class FeatureFileIndexingResult { + public FeatureVector[] v = null; + public int level = -1; + + public FeatureFileIndexingResult(FeatureVector[] setV, int setLevel) { + this.v = setV; + this.level = setLevel; + } + + public int[] getUnitIndexes() { + int[] ret = new int[v.length]; + for (int i = 0; i < v.length; i++) { + ret[i] = v[i].getUnitIndex(); + } + return (ret); + } +} diff --git a/marytts-unitselection/src/main/java/marytts/cart/impose/MaryNode.java b/marytts-unitselection/src/main/java/marytts/cart/impose/MaryNode.java new file mode 100644 index 0000000000..708d71e3eb --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/cart/impose/MaryNode.java @@ -0,0 +1,133 @@ +/** + * Copyright 2006 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.cart.impose; + +import marytts.features.FeatureVector; + +/** + * A generic node class for the tree structures. + * + * @author sacha + * + */ +public class MaryNode { + + protected int featureIndex = -1; + protected int from = 0; + protected int to = 0; + private MaryNode[] kids = null; + + /***************************/ + /* Constructor */ + public MaryNode(int setFrom, int setTo) { + from = setFrom; + to = setTo; + kids = null; + } + + /******************************/ + /* Getters for various fields */ + public int getFrom() { + return (from); + } + + public int getTo() { + return (to); + } + + public int getNumChildren() { + return (kids.length); + } + + public MaryNode[] getChildren() { + return (kids); + } + + /******************************/ + /* Feature index management */ + public void setFeatureIndex(int i) { + featureIndex = i; + } + + public int getFeatureIndex() { + return (featureIndex); + } + + /***************************/ + /* Node splitting */ + public void split(int numKids) { + kids = new MaryNode[numKids]; + } + + public void setChild(int i, MaryNode n) { + kids[i] = n; + } + + public MaryNode getChild(int i) { + return (kids[i]); + } + + /*************************************/ + /* Check if this is a node or a leaf */ + public boolean isNode() { + return (kids != null); + } + + public boolean isLeaf() { + return (kids == null); + } + + // debug output + public void toStandardOut(FeatureArrayIndexer ffi, int level) { + + String blanks = ""; + for (int i = 0; i < level; i++) + blanks += " "; + + if (kids != null) { + String featureName = ffi.getFeatureDefinition().getFeatureName(featureIndex); + System.out.println("Node " + featureName + " has " + (to - from) + " units divided into " + kids.length + + " branches."); + for (int i = 0; i < kids.length; i++) { + if (kids[i] != null) { + System.out.print(blanks + "Branch " + i + "/" + kids.length + " ( " + + ffi.getFeatureDefinition().getFeatureName(featureIndex) + " is " + + ffi.getFeatureDefinition().getFeatureValueAsString(featureIndex, i) + " )" + " -> "); + kids[i].toStandardOut(ffi, level + 1); + } else { + System.out.println(blanks + "Branch " + i + "/" + kids.length + " ( " + + ffi.getFeatureDefinition().getFeatureName(featureIndex) + " is " + + ffi.getFeatureDefinition().getFeatureValueAsString(featureIndex, i) + " )" + + " -> DEAD BRANCH (0 units)"); + } + } + } else { + // get the unit indices + FeatureVector[] fv = ffi.getFeatureVectors(from, to); + System.out.print("LEAF has " + (to - from) + " units : "); + for (int i = 0; i < fv.length; i++) { + System.out.print(fv[i].getUnitIndex() + " "); + } + System.out.print("\n"); + } + + } + +} diff --git a/marytts-unitselection/src/main/java/marytts/cart/io/DirectedGraphReader.java b/marytts-unitselection/src/main/java/marytts/cart/io/DirectedGraphReader.java new file mode 100644 index 0000000000..ec57b3f8d8 --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/cart/io/DirectedGraphReader.java @@ -0,0 +1,290 @@ +/** + * Copyright 2006 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.cart.io; + +import java.io.BufferedInputStream; +import java.io.ByteArrayInputStream; +import java.io.DataInput; +import java.io.DataInputStream; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Properties; + +import marytts.cart.DecisionNode; +import marytts.cart.DirectedGraph; +import marytts.cart.DirectedGraphNode; +import marytts.cart.LeafNode; +import marytts.cart.Node; +import marytts.exceptions.MaryConfigurationException; +import marytts.features.FeatureDefinition; +import marytts.util.data.MaryHeader; + +/** + * IO functions for Directed graphs in Mary format + * + * @author Marcela Charfuelan, Marc Schröder + */ +public class DirectedGraphReader { + /** Bit code for identifying a node id as a leaf node id in binary DirectedGraph files */ + public static int LEAFNODE = 0; + /** Bit code for identifying a node id as a decision node id in binary DirectedGraph files */ + public static int DECISIONNODE = 1; + /** Bit code for identifying a node id as a directed node id in binary DirectedGraph files */ + public static int DIRECTEDGRAPHNODE = 2; + + /** + * Load the directed graph from the given file + * + * @param fileName + * the file to load the cart from + * @param featDefinition + * the feature definition + * @param dummy + * unused, just here for compatibility with the FeatureFileIndexer. + * @throws IOException + * , {@link MaryConfigurationException} if a problem occurs while loading + */ + public DirectedGraph load(String fileName) throws IOException, MaryConfigurationException { + InputStream is = new FileInputStream(fileName); + try { + return load(is); + } finally { + is.close(); + } + } + + /** + * Load the directed graph from the given file + * + * @param fileName + * the file to load the cart from + * @param featDefinition + * the feature definition + * @param dummy + * unused, just here for compatibility with the FeatureFileIndexer. + * @throws IOException + * , {@link MaryConfigurationException} if a problem occurs while loading + */ + public DirectedGraph load(InputStream inStream) throws IOException, MaryConfigurationException { + BufferedInputStream buffInStream = new BufferedInputStream(inStream); + assert buffInStream.markSupported(); + buffInStream.mark(10000); + // open the CART-File and read the header + DataInput raf = new DataInputStream(buffInStream); + + MaryHeader maryHeader = new MaryHeader(raf); + if (!maryHeader.hasCurrentVersion()) { + throw new IOException("Wrong version of database file"); + } + if (maryHeader.getType() != MaryHeader.DIRECTED_GRAPH) { + if (maryHeader.getType() == MaryHeader.CARTS) { + buffInStream.reset(); + return new MaryCARTReader().loadFromStream(buffInStream); + } else { + throw new IOException("Not a directed graph file"); + } + } + + // Read properties + short propDataLength = raf.readShort(); + Properties props; + if (propDataLength == 0) { + props = null; + } else { + byte[] propsData = new byte[propDataLength]; + raf.readFully(propsData); + ByteArrayInputStream bais = new ByteArrayInputStream(propsData); + props = new Properties(); + props.load(bais); + bais.close(); + } + + // Read the feature definition + FeatureDefinition featureDefinition = new FeatureDefinition(raf); + + // read the decision nodes + int numDecNodes = raf.readInt(); // number of decision nodes + + // First we need to read all nodes into memory, then we can link them properly + // in terms of parent/child. + DecisionNode[] dns = new DecisionNode[numDecNodes]; + int[][] childIndexes = new int[numDecNodes][]; + for (int i = 0; i < numDecNodes; i++) { + // read one decision node + int featureNameIndex = raf.readInt(); + int nodeTypeNr = raf.readInt(); + DecisionNode.Type nodeType = DecisionNode.Type.values()[nodeTypeNr]; + int numChildren = 2; // for binary nodes + switch (nodeType) { + case BinaryByteDecisionNode: + int criterion = raf.readInt(); + dns[i] = new DecisionNode.BinaryByteDecisionNode(featureNameIndex, (byte) criterion, featureDefinition); + break; + case BinaryShortDecisionNode: + criterion = raf.readInt(); + dns[i] = new DecisionNode.BinaryShortDecisionNode(featureNameIndex, (short) criterion, featureDefinition); + break; + case BinaryFloatDecisionNode: + float floatCriterion = raf.readFloat(); + dns[i] = new DecisionNode.BinaryFloatDecisionNode(featureNameIndex, floatCriterion, featureDefinition); + break; + case ByteDecisionNode: + numChildren = raf.readInt(); + if (featureDefinition.getNumberOfValues(featureNameIndex) != numChildren) { + throw new IOException("Inconsistent cart file: feature " + featureDefinition.getFeatureName(featureNameIndex) + + " should have " + featureDefinition.getNumberOfValues(featureNameIndex) + + " values, but decision node " + i + " has only " + numChildren + " child nodes"); + } + dns[i] = new DecisionNode.ByteDecisionNode(featureNameIndex, numChildren, featureDefinition); + break; + case ShortDecisionNode: + numChildren = raf.readInt(); + if (featureDefinition.getNumberOfValues(featureNameIndex) != numChildren) { + throw new IOException("Inconsistent cart file: feature " + featureDefinition.getFeatureName(featureNameIndex) + + " should have " + featureDefinition.getNumberOfValues(featureNameIndex) + + " values, but decision node " + i + " has only " + numChildren + " child nodes"); + } + dns[i] = new DecisionNode.ShortDecisionNode(featureNameIndex, numChildren, featureDefinition); + } + dns[i].setUniqueDecisionNodeId(i + 1); + // now read the children, indexes only: + childIndexes[i] = new int[numChildren]; + for (int k = 0; k < numChildren; k++) { + childIndexes[i][k] = raf.readInt(); + } + } + + // read the leaves + int numLeafNodes = raf.readInt(); // number of leaves, it does not include empty leaves + LeafNode[] lns = new LeafNode[numLeafNodes]; + + for (int j = 0; j < numLeafNodes; j++) { + // read one leaf node + int leafTypeNr = raf.readInt(); + LeafNode.LeafType leafNodeType = LeafNode.LeafType.values()[leafTypeNr]; + switch (leafNodeType) { + case IntArrayLeafNode: + int numData = raf.readInt(); + int[] data = new int[numData]; + for (int d = 0; d < numData; d++) { + data[d] = raf.readInt(); + } + lns[j] = new LeafNode.IntArrayLeafNode(data); + break; + case FloatLeafNode: + float stddev = raf.readFloat(); + float mean = raf.readFloat(); + lns[j] = new LeafNode.FloatLeafNode(new float[] { stddev, mean }); + break; + case IntAndFloatArrayLeafNode: + case StringAndFloatLeafNode: + int numPairs = raf.readInt(); + int[] ints = new int[numPairs]; + float[] floats = new float[numPairs]; + for (int d = 0; d < numPairs; d++) { + ints[d] = raf.readInt(); + floats[d] = raf.readFloat(); + } + if (leafNodeType == LeafNode.LeafType.IntAndFloatArrayLeafNode) + lns[j] = new LeafNode.IntAndFloatArrayLeafNode(ints, floats); + else + lns[j] = new LeafNode.StringAndFloatLeafNode(ints, floats); + break; + case FeatureVectorLeafNode: + throw new IllegalArgumentException("Reading feature vector leaf nodes is not yet implemented"); + case PdfLeafNode: + throw new IllegalArgumentException("Reading pdf leaf nodes is not yet implemented"); + } + lns[j].setUniqueLeafId(j + 1); + } + + // Graph nodes + int numDirectedGraphNodes = raf.readInt(); + DirectedGraphNode[] graphNodes = new DirectedGraphNode[numDirectedGraphNodes]; + int[] dgnLeafIndices = new int[numDirectedGraphNodes]; + int[] dgnDecIndices = new int[numDirectedGraphNodes]; + for (int g = 0; g < numDirectedGraphNodes; g++) { + graphNodes[g] = new DirectedGraphNode(null, null); + graphNodes[g].setUniqueGraphNodeID(g + 1); + dgnLeafIndices[g] = raf.readInt(); + dgnDecIndices[g] = raf.readInt(); + } + + // Now, link up the decision nodes with their daughters + for (int i = 0; i < numDecNodes; i++) { + // System.out.print(dns[i]+" "+dns[i].getFeatureName()+" "); + for (int k = 0; k < childIndexes[i].length; k++) { + Node child = childIndexToNode(childIndexes[i][k], dns, lns, graphNodes); + dns[i].addDaughter(child); + // System.out.print(" "+dns[i].getDaughter(k)); + } + // System.out.println(); + } + // And link up directed graph nodes + for (int g = 0; g < numDirectedGraphNodes; g++) { + Node leaf = childIndexToNode(dgnLeafIndices[g], dns, lns, graphNodes); + graphNodes[g].setLeafNode(leaf); + Node dec = childIndexToNode(dgnDecIndices[g], dns, lns, graphNodes); + if (dec != null && !dec.isDecisionNode()) + throw new IllegalArgumentException("Only decision nodes allowed, read " + dec.getClass()); + graphNodes[g].setDecisionNode((DecisionNode) dec); + // System.out.println("Graph node "+(g+1)+", leaf: "+Integer.toHexString(dgnLeafIndices[g])+", "+leaf+" -- dec: "+Integer.toHexString(dgnDecIndices[g])+", "+dec); + } + + Node rootNode; + if (graphNodes.length > 0) { + rootNode = graphNodes[0]; + } else if (dns.length > 0) { + rootNode = dns[0]; + // CART behaviour, not sure if this is needed: + // Now count all data once, so that getNumberOfData() + // will return the correct figure. + ((DecisionNode) rootNode).countData(); + } else if (lns.length > 0) { + rootNode = lns[0]; // single-leaf tree... + } else { + rootNode = null; + } + + // set the rootNode as the rootNode of cart + return new DirectedGraph(rootNode, featureDefinition, props); + } + + private Node childIndexToNode(int childIndexAndType, DecisionNode[] dns, LeafNode[] lns, DirectedGraphNode[] graphNodes) { + int childIndex = childIndexAndType & 0x3fffffff; // the lower 30 bits + int childType = (childIndexAndType >> 30) & 0x03; // the highest two bits + if (childIndex == 0) { // an empty leaf + return null; + } else if (childType == DECISIONNODE) { // a decision node + assert childIndex - 1 < dns.length; + return dns[childIndex - 1]; + } else if (childType == LEAFNODE) { // a leaf node + assert childIndex - 1 < lns.length; + return lns[childIndex - 1]; + } else if (childType == DIRECTEDGRAPHNODE) { + assert childIndex - 1 < graphNodes.length; + return graphNodes[childIndex - 1]; + } else { + throw new IllegalArgumentException("Unexpected child type: " + childType); + } + + } +} diff --git a/marytts-unitselection/src/main/java/marytts/cart/io/DirectedGraphWriter.java b/marytts-unitselection/src/main/java/marytts/cart/io/DirectedGraphWriter.java new file mode 100644 index 0000000000..215d254098 --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/cart/io/DirectedGraphWriter.java @@ -0,0 +1,409 @@ +/** + * Copyright 2006 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.cart.io; + +import java.io.BufferedOutputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataOutput; +import java.io.DataOutputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.PrintWriter; +import java.util.Properties; + +import marytts.cart.DecisionNode; +import marytts.cart.DirectedGraph; +import marytts.cart.DirectedGraphNode; +import marytts.cart.LeafNode; +import marytts.cart.Node; +import marytts.cart.DecisionNode.BinaryByteDecisionNode; +import marytts.cart.DecisionNode.BinaryFloatDecisionNode; +import marytts.cart.DecisionNode.BinaryShortDecisionNode; +import marytts.cart.LeafNode.FeatureVectorLeafNode; +import marytts.cart.LeafNode.FloatLeafNode; +import marytts.cart.LeafNode.IntAndFloatArrayLeafNode; +import marytts.cart.LeafNode.IntArrayLeafNode; +import marytts.cart.LeafNode.LeafType; +import marytts.features.FeatureVector; +import marytts.util.MaryUtils; +import marytts.util.data.MaryHeader; + +import org.apache.log4j.Logger; + +/** + * IO functions for directed graphs in Mary format + * + * @author Marcela Charfuelan, Marc Schröder + */ +public class DirectedGraphWriter { + + protected Logger logger = MaryUtils.getLogger(this.getClass().getName()); + + /** + * Dump the graph in Mary format + * + * @param destDir + * the destination directory + */ + public void saveGraph(DirectedGraph graph, String destFile) throws IOException { + if (graph == null) + throw new NullPointerException("Cannot dump null graph"); + if (destFile == null) + throw new NullPointerException("No destination file"); + + logger.debug("Dumping directed graph in Mary format to " + destFile + " ..."); + + // Open the destination file and output the header + DataOutputStream out = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(destFile))); + // create new CART-header and write it to output file + MaryHeader hdr = new MaryHeader(MaryHeader.DIRECTED_GRAPH); + hdr.writeTo(out); + + Properties props = graph.getProperties(); + if (props == null) { + out.writeShort(0); + } else { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + props.store(baos, null); + byte[] propData = baos.toByteArray(); + out.writeShort(propData.length); + out.write(propData); + } + + // feature definition + graph.getFeatureDefinition().writeBinaryTo(out); + + // dump graph + dumpBinary(graph, out); + + // finish + out.close(); + logger.debug(" ... done\n"); + } + + public void toTextOut(DirectedGraph graph, PrintWriter pw) throws IOException { + try { + int numLeafNodes = setUniqueLeafNodeIds(graph); + int numDecNodes = setUniqueDecisionNodeIds(graph); + int numGraphNodes = setUniqueDirectedGraphNodeIds(graph); + pw.println("Num decision nodes= " + numDecNodes + " Num leaf nodes= " + numLeafNodes + + " Num directed graph nodes= " + numGraphNodes); + printDecisionNodes(graph, null, pw); + pw.println("\n----------------\n"); + printLeafNodes(graph, null, pw); + pw.println("\n----------------\n"); + printDirectedGraphNodes(graph, null, pw); + + pw.flush(); + pw.close(); + } catch (IOException ioe) { + IOException newIOE = new IOException("Error dumping graph to standard output"); + newIOE.initCause(ioe); + throw newIOE; + } + } + + /** + * Assign unique ids to leaf nodes. + * + * @param graph + * @return the number of different leaf nodes + */ + private int setUniqueLeafNodeIds(DirectedGraph graph) { + int i = 0; + for (LeafNode l : graph.getLeafNodes()) { + l.setUniqueLeafId(++i); + } + return i; + } + + /** + * Assign unique ids to decision nodes. + * + * @param graph + * @return the number of different decision nodes + */ + private int setUniqueDecisionNodeIds(DirectedGraph graph) { + int i = 0; + for (DecisionNode d : graph.getDecisionNodes()) { + d.setUniqueDecisionNodeId(++i); + } + return i; + } + + /** + * Assign unique ids to directed graph nodes. + * + * @param graph + * @return the number of different directed graph nodes + */ + private int setUniqueDirectedGraphNodeIds(DirectedGraph graph) { + int i = 0; + for (DirectedGraphNode g : graph.getDirectedGraphNodes()) { + g.setUniqueGraphNodeID(++i); + } + return i; + } + + private void dumpBinary(DirectedGraph graph, DataOutput os) throws IOException { + try { + int numLeafNodes = setUniqueLeafNodeIds(graph); + int numDecNodes = setUniqueDecisionNodeIds(graph); + int numGraphNodes = setUniqueDirectedGraphNodeIds(graph); + int maxNum = 1 << 30; + if (numLeafNodes > maxNum || numDecNodes > maxNum || numGraphNodes > maxNum) { + throw new UnsupportedOperationException("Cannot write more than " + maxNum + " nodes of one type in this format"); + } + // write the number of decision nodes + os.writeInt(numDecNodes); + printDecisionNodes(graph, os, null); + + // write the number of leaves. + os.writeInt(numLeafNodes); + printLeafNodes(graph, os, null); + + // write the number of directed graph nodes + os.writeInt(numGraphNodes); + printDirectedGraphNodes(graph, os, null); + + } catch (IOException ioe) { + IOException newIOE = new IOException("Error dumping CART to output stream"); + newIOE.initCause(ioe); + throw newIOE; + } + } + + private void printDecisionNodes(DirectedGraph graph, DataOutput out, PrintWriter pw) throws IOException { + for (DecisionNode decNode : graph.getDecisionNodes()) { + int id = decNode.getUniqueDecisionNodeId(); + String nodeDefinition = decNode.getNodeDefinition(); + int featureIndex = decNode.getFeatureIndex(); + DecisionNode.Type nodeType = decNode.getDecisionNodeType(); + + if (out != null) { + // dump in binary form to output + out.writeInt(featureIndex); + out.writeInt(nodeType.ordinal()); + // Now, questionValue, which depends on nodeType + switch (nodeType) { + case BinaryByteDecisionNode: + out.writeInt(((BinaryByteDecisionNode) decNode).getCriterionValueAsByte()); + assert decNode.getNumberOfDaugthers() == 2; + break; + case BinaryShortDecisionNode: + out.writeInt(((BinaryShortDecisionNode) decNode).getCriterionValueAsShort()); + assert decNode.getNumberOfDaugthers() == 2; + break; + case BinaryFloatDecisionNode: + out.writeFloat(((BinaryFloatDecisionNode) decNode).getCriterionValueAsFloat()); + assert decNode.getNumberOfDaugthers() == 2; + break; + case ByteDecisionNode: + case ShortDecisionNode: + out.writeInt(decNode.getNumberOfDaugthers()); + } + + // The child nodes + for (int i = 0, n = decNode.getNumberOfDaugthers(); i < n; i++) { + Node daughter = decNode.getDaughter(i); + if (daughter == null) { + out.writeInt(0); + } else if (daughter.isDecisionNode()) { + int daughterID = ((DecisionNode) daughter).getUniqueDecisionNodeId(); + // Mark as decision node: + daughterID |= DirectedGraphReader.DECISIONNODE << 30; + out.writeInt(daughterID); + } else if (daughter.isLeafNode()) { + int daughterID = ((LeafNode) daughter).getUniqueLeafId(); + // Mark as leaf node: + if (daughterID != 0) + daughterID |= DirectedGraphReader.LEAFNODE << 30; + out.writeInt(daughterID); + } else if (daughter.isDirectedGraphNode()) { + int daughterID = ((DirectedGraphNode) daughter).getUniqueGraphNodeID(); + // Mark as directed graph node: + if (daughterID != 0) + daughterID |= DirectedGraphReader.DIRECTEDGRAPHNODE << 30; + out.writeInt(daughterID); + } + } + } + if (pw != null) { + // dump to print writer + StringBuilder strNode = new StringBuilder("-" + id + " " + nodeDefinition); + for (int i = 0, n = decNode.getNumberOfDaugthers(); i < n; i++) { + strNode.append(" "); + Node daughter = decNode.getDaughter(i); + if (daughter == null) { + strNode.append("0"); + } else if (daughter.isDecisionNode()) { + int daughterID = ((DecisionNode) daughter).getUniqueDecisionNodeId(); + strNode.append("-").append(daughterID); + out.writeInt(daughterID); + } else if (daughter.isLeafNode()) { + int daughterID = ((LeafNode) daughter).getUniqueLeafId(); + if (daughterID == 0) + strNode.append("0"); + else + strNode.append("id").append(daughterID); + } else if (daughter.isDirectedGraphNode()) { + int daughterID = ((DirectedGraphNode) daughter).getUniqueGraphNodeID(); + if (daughterID == 0) + strNode.append("0"); + else + strNode.append("DGN").append(daughterID); + } + } + pw.println(strNode.toString()); + } + } + } + + private void printLeafNodes(DirectedGraph graph, DataOutput out, PrintWriter pw) throws IOException { + for (LeafNode leaf : graph.getLeafNodes()) { + if (leaf.getUniqueLeafId() == 0) // empty leaf, do not write + continue; + LeafType leafType = leaf.getLeafNodeType(); + if (leafType == LeafType.FeatureVectorLeafNode) { + leafType = LeafType.IntArrayLeafNode; + // save feature vector leaf nodes as int array leaf nodes + } + if (out != null) { + // Leaf node type + out.writeInt(leafType.ordinal()); + } + if (pw != null) { + pw.print("id" + leaf.getUniqueLeafId() + " " + leafType); + } + switch (leaf.getLeafNodeType()) { + case IntArrayLeafNode: + int data[] = ((IntArrayLeafNode) leaf).getIntData(); + // Number of data points following: + if (out != null) + out.writeInt(data.length); + if (pw != null) + pw.print(" " + data.length); + // for each index, write the index + for (int i = 0; i < data.length; i++) { + if (out != null) + out.writeInt(data[i]); + if (pw != null) + pw.print(" " + data[i]); + } + break; + case FloatLeafNode: + float stddev = ((FloatLeafNode) leaf).getStDeviation(); + float mean = ((FloatLeafNode) leaf).getMean(); + if (out != null) { + out.writeFloat(stddev); + out.writeFloat(mean); + } + if (pw != null) { + pw.print(" 1 " + stddev + " " + mean); + } + break; + case IntAndFloatArrayLeafNode: + case StringAndFloatLeafNode: + int data1[] = ((IntAndFloatArrayLeafNode) leaf).getIntData(); + float floats[] = ((IntAndFloatArrayLeafNode) leaf).getFloatData(); + // Number of data points following: + if (out != null) + out.writeInt(data1.length); + if (pw != null) + pw.print(" " + data1.length); + // for each index, write the index and then its float + for (int i = 0; i < data1.length; i++) { + if (out != null) { + out.writeInt(data1[i]); + out.writeFloat(floats[i]); + } + if (pw != null) + pw.print(" " + data1[i] + " " + floats[i]); + } + break; + case FeatureVectorLeafNode: + FeatureVector fv[] = ((FeatureVectorLeafNode) leaf).getFeatureVectors(); + // Number of data points following: + if (out != null) + out.writeInt(fv.length); + if (pw != null) + pw.print(" " + fv.length); + // for each feature vector, write the index + for (int i = 0; i < fv.length; i++) { + if (out != null) + out.writeInt(fv[i].getUnitIndex()); + if (pw != null) + pw.print(" " + fv[i].getUnitIndex()); + } + break; + case PdfLeafNode: + throw new IllegalArgumentException("Writing of pdf leaf nodes not yet implemented"); + } + if (pw != null) + pw.println(); + } + } + + private void printDirectedGraphNodes(DirectedGraph graph, DataOutput out, PrintWriter pw) throws IOException { + for (DirectedGraphNode g : graph.getDirectedGraphNodes()) { + int id = g.getUniqueGraphNodeID(); + if (id == 0) + continue;// empty node, do not write + Node leaf = g.getLeafNode(); + int leafID = 0; + int leafNodeType = 0; + if (leaf != null) { + if (leaf instanceof LeafNode) { + leafID = ((LeafNode) leaf).getUniqueLeafId(); + leafNodeType = DirectedGraphReader.LEAFNODE; + } else if (leaf instanceof DirectedGraphNode) { + leafID = ((DirectedGraphNode) leaf).getUniqueGraphNodeID(); + leafNodeType = DirectedGraphReader.DIRECTEDGRAPHNODE; + } else { + throw new IllegalArgumentException("Unexpected leaf type: " + leaf.getClass()); + } + } + DecisionNode d = g.getDecisionNode(); + int decID = d != null ? d.getUniqueDecisionNodeId() : 0; + if (out != null) { + int outLeafId = leafID == 0 ? 0 : leafID | (leafNodeType << 30); + out.writeInt(outLeafId); + int outDecId = decID == 0 ? 0 : decID | (DirectedGraphReader.DECISIONNODE << 30); + out.writeInt(outDecId); + } + if (pw != null) { + pw.print("DGN" + id); + if (leafID == 0) { + pw.print(" 0"); + } else if (leaf.isLeafNode()) { + pw.print(" id" + leafID); + } else { + assert leaf.isDirectedGraphNode(); + pw.print(" DGN" + leafID); + } + if (decID == 0) + pw.print(" 0"); + else + pw.print(" -" + decID); + pw.println(); + } + } + } +} diff --git a/marytts-unitselection/src/main/java/marytts/cart/io/HTSCARTReader.java b/marytts-unitselection/src/main/java/marytts/cart/io/HTSCARTReader.java new file mode 100644 index 0000000000..fba530ebfc --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/cart/io/HTSCARTReader.java @@ -0,0 +1,556 @@ +/** + * The HMM-Based Speech Synthesis System (HTS) + * HTS Working Group + * + * Department of Computer Science + * Nagoya Institute of Technology + * and + * Interdisciplinary Graduate School of Science and Engineering + * Tokyo Institute of Technology + * + * Portions Copyright (c) 2001-2006 + * All Rights Reserved. + * + * Portions Copyright 2000-2007 DFKI GmbH. + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to use and + * distribute this software and its documentation without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of this work, and to permit persons to whom this + * work is furnished to do so, subject to the following conditions: + * + * 1. The source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Any modifications to the source code must be clearly + * marked as such. + * + * 3. Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the + * following disclaimer in the documentation and/or other + * materials provided with the distribution. Otherwise, one + * must contact the HTS working group. + * + * NAGOYA INSTITUTE OF TECHNOLOGY, TOKYO INSTITUTE OF TECHNOLOGY, + * HTS WORKING GROUP, AND THE CONTRIBUTORS TO THIS WORK DISCLAIM + * ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING ALL + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO EVENT + * SHALL NAGOYA INSTITUTE OF TECHNOLOGY, TOKYO INSTITUTE OF + * TECHNOLOGY, HTS WORKING GROUP, NOR THE CONTRIBUTORS BE LIABLE + * FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY + * DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, + * WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTUOUS + * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR + * PERFORMANCE OF THIS SOFTWARE. + * + */ +package marytts.cart.io; + +import java.io.BufferedInputStream; +import java.io.BufferedReader; +import java.io.DataInputStream; +import java.io.FileInputStream; +import java.io.FileReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.StringReader; +import java.util.Scanner; +import java.util.StringTokenizer; + +import marytts.cart.CART; +import marytts.cart.DecisionNode; +import marytts.cart.LeafNode; +import marytts.cart.Node; +import marytts.cart.DecisionNode.BinaryByteDecisionNode; +import marytts.cart.LeafNode.PdfLeafNode; +import marytts.exceptions.MaryConfigurationException; +import marytts.features.FeatureDefinition; +import marytts.htsengine.PhoneTranslator; +import marytts.htsengine.HMMData.PdfFileFormat; +import marytts.util.MaryUtils; + +import org.apache.log4j.Logger; + +/** + * Reader functions for CARTs in HTS format + * + * @author Marcela Charfuelan + */ +public class HTSCARTReader { + + private FeatureDefinition featDef; + private PhoneTranslator phTrans; + private Logger logger = MaryUtils.getLogger("HTSCARTReader"); + private int vectorSize; // the vector size of the mean and variance on the leaves of the tree. + + public int getVectorSize() { + return vectorSize; + } + + /** + * Load the cart from the given file + * + * @param nummStates + * number of states in the HTS model, it will create one cart tree per state. + * @param treefileName + * the HTS tree text file, example tree-mgc.inf. + * @param pdfFileName + * the corresponding HTS pdf binary file, example mgc.pdf. + * @param featDefinition + * the feature definition + * @param CART + * [] Fills this array of CART trees, one per state. + * @return the size of the mean and variance vectors on the leaves. + * @throws IOException + * if a problem occurs while loading + */ + public CART[] load(int numStates, InputStream treeStream, InputStream pdfStream, PdfFileFormat fileFormat, + FeatureDefinition featDefinition, PhoneTranslator phTranslator) throws IOException, MaryConfigurationException { + + featDef = featDefinition; + // phTrans = phoneTranslator; + int i, j, length, state; + BufferedReader s = null; + String line, aux; + + phTrans = phTranslator; + + // create the number of carts it is going to read + CART treeSet[] = new CART[numStates]; + for (i = 0; i < numStates; i++) + treeSet[i] = new CART(); + + // First load pdfs, so when creates the tree fill the leaf nodes with + // the corresponding mean and variances. + /** + * load pdf's, mean and variance pdfs format : pdf[numStates][numPdfs][numStreams][2*vectorSize] + * ------------------------------------------------------------------- for dur : pdf[ 1 ][numPdfs][ 1 ][2*numStates ] for + * mgc,str,mag: pdf[numStates][numPdfs][ 1 ][2*vectorSize]; for joinModel : pdf[ 1 ][numPdfs][ 1 ][2*vectorSize]; for lf0 + * : pdf[numStates][numPdfs][numStreams][ 4 ] for gv-switch : pdf[ 1 ][ 1 ][ 1 ][ 1 ] + * ------------------------------------------------------------------ - numPdf : corresponds to the unique leaf node id. - + * 2*vectorSize : means that mean and variance are in the same vector. - 4 in lf0 : means 0: mean, 1: variance, 2: voiced + * weight and 3: unvoiced weight ------------------------------------------------------------------ + */ + double pdf[][][][]; + pdf = loadPdfs(numStates, pdfStream, fileFormat); + + assert featDefinition != null : "Feature Definition was not set"; + + /* read lines of tree-*.inf fileName */ + s = new BufferedReader(new InputStreamReader(treeStream, "UTF-8")); + + // skip questions section + while ((line = s.readLine()) != null) { + if (line.indexOf("QS") < 0) + break; /* a new state is indicated by {*}[2], {*}[3], ... */ + } + + while ((line = s.readLine()) != null) { + if (line.indexOf("{*}") >= 0) { /* this is the indicator of a new state-tree */ + aux = line.substring(line.indexOf("[") + 1, line.indexOf("]")); + state = Integer.parseInt(aux); + // loads one cart tree per state + treeSet[state - 2].setRootNode(loadStateTree(s, pdf[state - 2])); + + // Now count all data once, so that getNumberOfData() + // will return the correct figure. + if (treeSet[state - 2].getRootNode() instanceof DecisionNode) + ((DecisionNode) treeSet[state - 2].getRootNode()).countData(); + + logger.debug("load: CART[" + (state - 2) + "], total number of nodes in this CART: " + + treeSet[state - 2].getNumNodes()); + } + } /* while */ + if (s != null) + s.close(); + + /* check that the tree was correctly loaded */ + if (treeSet.length == 0) { + throw new IOException("LoadTreeSet: error no trees loaded"); + } + + return treeSet; + + } + + /** + * Load a tree per state + * + * @param s + * : text scanner of the whole tree-*.inf file + * @param pdf + * : the pdfs for this state, pdf[numPdfs][numStreams][2*vectorSize] + */ + private Node loadStateTree(BufferedReader s, double pdf[][][]) throws IOException, MaryConfigurationException { + + Node rootNode = null; + Node lastNode = null; + + StringTokenizer sline; + String aux, buf; + + // create an empty binary decision node with unique id=0, this will be the rootNode + Node nextNode = new DecisionNode.BinaryByteDecisionNode(0, featDef); + + // this is the rootNode + rootNode = nextNode; + nextNode.setIsRoot(true); + + int iaux, feaIndex, ndec, nleaf; + ndec = 0; + nleaf = 0; + Node node = null; + aux = s.readLine(); /* next line for this state tree must be { */ + int id; + + if (aux.indexOf("{") >= 0) { + while ((aux = s.readLine()) != null && aux.indexOf("}") < 0) { /* last line for this state tree must be } */ + /* then parse this line, it contains 4 fields */ + /* 1: node index # 2: Question name 3: NO # node 4: YES # node */ + sline = new StringTokenizer(aux); + + /* 1: gets index node and looks for the node whose idx = buf */ + buf = sline.nextToken(); + if (buf.startsWith("-")) { + id = Integer.parseInt(buf.substring(1)); + ndec++; + } else if (buf.contentEquals("0")) + id = 0; + else + throw new MaryConfigurationException("LoadStateTree: line does not start with a decision node (-id), line=" + + aux); + // 1. find the node in the tree, it has to be already created. + node = findDecisionNode(rootNode, id); + + if (node == null) + throw new MaryConfigurationException("LoadStateTree: Node not found, index = " + buf); + else { + /* 2: gets question name and question name val */ + buf = sline.nextToken(); + String[] fea_val = buf.split("="); /* splits featureName=featureValue */ + feaIndex = featDef.getFeatureIndex(fea_val[0]); + + /* Replace back punctuation values */ + /* what about tricky phones, if using halfphones it would not be necessary */ + if (fea_val[0].contentEquals("sentence_punc") || fea_val[0].contentEquals("prev_punctuation") + || fea_val[0].contentEquals("next_punctuation")) { + // System.out.print("CART replace punc: " + fea_val[0] + " = " + fea_val[1]); + fea_val[1] = phTrans.replaceBackPunc(fea_val[1]); + // System.out.println(" --> " + fea_val[0] + " = " + fea_val[1]); + } else if (fea_val[0].contains("tobi_")) { + // System.out.print("CART replace tobi: " + fea_val[0] + " = " + fea_val[1]); + fea_val[1] = phTrans.replaceBackToBI(fea_val[1]); + // System.out.println(" --> " + fea_val[0] + " = " + fea_val[1]); + } else if (fea_val[0].contains("phone")) { + // System.out.print("CART replace phone: " + fea_val[0] + " = " + fea_val[1]); + fea_val[1] = phTrans.replaceBackTrickyPhones(fea_val[1]); + // System.out.println(" --> " + fea_val[0] + " = " + fea_val[1]); + } + + // add featureName and featureValue to the decision nod + ((BinaryByteDecisionNode) node).setFeatureAndFeatureValue(fea_val[0], fea_val[1]); + + // add NO and YES indexes to the daughther nodes + /* NO index */ + buf = sline.nextToken(); + if (buf.startsWith("-")) { // Decision node + iaux = Integer.parseInt(buf.substring(1)); + // create an empty binary decision node with unique id + BinaryByteDecisionNode auxnode = new DecisionNode.BinaryByteDecisionNode(iaux, featDef); + ((DecisionNode) node).replaceDaughter(auxnode, 1); + } else { // LeafNode + iaux = Integer.parseInt(buf.substring(buf.lastIndexOf("_") + 1, buf.length() - 1)); + // create an empty PdfLeafNode + PdfLeafNode auxnode = new LeafNode.PdfLeafNode(iaux, pdf[iaux - 1]); + ((DecisionNode) node).replaceDaughter(auxnode, 1); + nleaf++; + } + + /* YES index */ + buf = sline.nextToken(); + if (buf.startsWith("-")) { // Decision node + iaux = Integer.parseInt(buf.substring(1)); + // create an empty binary decision node with unique id=0 + BinaryByteDecisionNode auxnode = new DecisionNode.BinaryByteDecisionNode(iaux, featDef); + ((DecisionNode) node).replaceDaughter(auxnode, 0); + } else { // LeafNode + iaux = Integer.parseInt(buf.substring(buf.lastIndexOf("_") + 1, buf.length() - 1)); + // create an empty PdfLeafNode + PdfLeafNode auxnode = new LeafNode.PdfLeafNode(iaux, pdf[iaux - 1]); + ((DecisionNode) node).replaceDaughter(auxnode, 0); + nleaf++; + } + } /* if node not null */ + sline = null; + } /* while there is another line and the line does not contain } */ + } /* if not "{" */ + + logger.debug("loadStateTree: loaded CART contains " + (ndec + 1) + " Decision nodes and " + nleaf + " Leaf nodes."); + return rootNode; + + } /* method loadTree() */ + + /** + * @param node + * , decision node + * @param numId + * , index to look for. + * @return + */ + private Node findDecisionNode(Node node, int numId) { + + Node aux = null; + + if (node instanceof DecisionNode) { + // System.out.print(" id=" + ((DecisionNode)node).getUniqueDecisionNodeId()); + if (((DecisionNode) node).getUniqueDecisionNodeId() == numId) + return node; + else { + for (int i = 0; i < ((DecisionNode) node).getNumberOfDaugthers(); i++) { + aux = findDecisionNode(((DecisionNode) node).getDaughter(i), numId); + if (aux != null) + return aux; + } + } + } + return aux; + + } /* method findDecisionNode */ + + /** + * Load pdf's, mean and variance the #leaves corresponds to the unique leaf node id pdf --> + * [#states][#leaves][#streams][vectorsize] The format of pdf files for mgc, str or mag is: header: 4 byte int: dimension + * feature vector 4 byte int: # of leaf nodes for state 1 4 byte int: # of leaf nodes for state 2 ... 4 byte int: # of leaf + * nodes for state N probability distributions: 4 byte float means and variances (2*pdfVsize): all leaves for state 1 4 byte + * float means and variances (2*pdfVsize): all leaves for state 2 ... 4 byte float means and variances (2*pdfVsize): all + * leaves for state N --------------------------------------------------------------------- The format of pdf files for dur + * and JoinModeller is: header: 4 byte int: # of HMM states <-- this is the dimension of vector in duration 4 byte int: # of + * leaf nodes for state 1 <-- dur has just one state probability distributions: 4 byte float means and variances (2*HMMsize): + * all leaves for state 1 --------------------------------------------------------------------- The format of pdf files for + * lf0 is: header: 4 byte int: dimension feature vector 4 byte int: # of leaf nodes for state 1 4 byte int: # of leaf nodes + * for state 2 ... 4 byte int: # of leaf nodes for state N probability distributions: 4 byte float mean, variance, voiced, + * unvoiced (4 floats): stream 1..S, leaf 1..L, state 1 4 byte float mean, variance, voiced, unvoiced (4 floats): stream 1..S, + * leaf 1..L, state 2 ... 4 byte float mean, variance, voiced, unvoiced (4 floats): stream 1..S, leaf 1..L, state N + */ + private double[][][][] loadPdfs(int numState, InputStream pdfStream, PdfFileFormat fileFormat) throws IOException, + MaryConfigurationException { + + DataInputStream data_in; + int i, j, k, l, numDurPdf, lf0Stream; + double vw, uvw; + int vsize; + int numPdf[]; + int numStream; + int numMSDFlag; /* MSD: Multi stream dimensions: in case of lf0 for example */ + double pdf[][][][] = null; // pdf[numState][numPdf][stream][vsize]; + + // TODO: how to make this loading more general, different files have different formats. Right now the way + // of loading depends on the name of the file, I need to change that! + // pdfFileName.contains("dur.pdf") || pdfFileName.contains("joinModeller.pdf") + if (fileFormat == PdfFileFormat.dur || fileFormat == PdfFileFormat.join) { + /* ________________________________________________________________ */ + /*-------------------- load pdfs for duration --------------------*/ + data_in = new DataInputStream(new BufferedInputStream(pdfStream)); + logger.debug("loadPdfs reading model of type " + fileFormat); + + /* read the number of states & the number of pdfs (leaf nodes) */ + /* read the number of HMM states, this number is the same for all pdf's. */ + + numMSDFlag = data_in.readInt(); + numStream = data_in.readInt(); + vectorSize = data_in.readInt(); + // ---vectorSize = numState; + // System.out.println("loadPdfs: nstate = " + nstate); + + numState = numStream; + + /* check number of states */ + if (numState < 0) + throw new MaryConfigurationException("loadPdfs: #HMM states must be positive value."); + + /* read the number of duration pdfs */ + numDurPdf = data_in.readInt(); + logger.debug("loadPdfs: numPdf[state:0]=" + numDurPdf); + + /* Now we know the number of duration pdfs and the vector size which is */ + /* the number of states in each HMM. Here the vector size is 2*nstate because */ + /* the first nstate correspond to the mean and the second nstate correspond */ + /* to the diagonal variance vector, the mean and variance are copied here in */ + /* only one vector. */ + /* 2*nstate because the vector size for duration is the number of states */ + pdf = new double[1][numDurPdf][1][2 * numState]; // just one state and one stream + vsize = (2 * numState); + /* read pdfs (mean & variance) */ + // NOTE: Here (hts_engine v1.04) the order is different as before, here mean and variance are saved consecutively + for (i = 0; i < numDurPdf; i++) { + for (j = 0; j < numState; j++) { + pdf[0][i][0][j] = data_in.readFloat(); // read mean + pdf[0][i][0][j + numState] = data_in.readFloat(); // read variance + // System.out.println("durpdf[" + i + "]" + "[" + j + "]:" + pdf[0][i][0][j]); + } + } + data_in.close(); + data_in = null; + + } else if (fileFormat == PdfFileFormat.lf0) { // pdfFileName.contains("lf0.pdf") + /* ____________________________________________________________________ */ + /*-------------------- load pdfs for Log F0 --------------*/ + data_in = new DataInputStream(new BufferedInputStream(pdfStream)); + logger.debug("loadPdfs reading model of type " + fileFormat); + /* read the number of streams for f0 modeling */ + // lf0Stream = data_in.readInt(); + // vectorSize = lf0Stream; + numMSDFlag = data_in.readInt(); + numStream = data_in.readInt(); + vectorSize = data_in.readInt(); + + lf0Stream = numStream; + // System.out.println("loadPdfs: lf0stream = " + lf0stream); + + if (lf0Stream < 0) + throw new MaryConfigurationException("loadPdfs: #stream for log f0 part must be positive value."); + + /* read the number of pdfs for each state position */ + pdf = new double[numState][][][]; + numPdf = new int[numState]; + for (i = 0; i < numState; i++) { + numPdf[i] = data_in.readInt(); + logger.debug("loadPdfs: numPdf[state:" + i + "]=" + numPdf[i]); + if (numPdf[i] < 0) + throw new MaryConfigurationException("loadPdfs: #lf0 pdf at state " + i + " must be positive value."); + // System.out.println("nlf0pdf[" + i + "] = " + numPdf[i]); + /* Now i know the size of pdfs for lf0 [#states][#leaves][#streams][lf0_vectorsize] */ + /* lf0_vectorsize = 4: mean, variance, voiced weight, and unvoiced weight */ + /* so i can allocate memory for lf0pdf[][][] */ + pdf[i] = new double[numPdf[i]][lf0Stream][4]; + } + + /* read lf0 pdfs (mean, variance and weight). */ + for (i = 0; i < numState; i++) { + for (j = 0; j < numPdf[i]; j++) { + for (k = 0; k < lf0Stream; k++) { + for (l = 0; l < 4; l++) { + pdf[i][j][k][l] = data_in.readFloat(); + // System.out.format("pdf[%d][%d][%d][%d]=%f\n", i,j,k,l,pdf[i][j][k][l]); + } + // System.out.format("\n"); + // NOTE: Here (hts_engine v1.04) the order seem to be the same as before + /* pdf[i][j][k][0]; mean */ + /* pdf[i][j][k][1]; vari */ + vw = pdf[i][j][k][2]; /* voiced weight */ + uvw = pdf[i][j][k][3]; /* unvoiced weight */ + if (vw < 0.0 || uvw < 0.0 || vw + uvw < 0.99 || vw + uvw > 1.01) + throw new MaryConfigurationException("loadPdfs: voiced/unvoiced weights must be within 0.99 to 1.01."); + } + } + } + + data_in.close(); + data_in = null; + + } else if (fileFormat == PdfFileFormat.mgc || fileFormat == PdfFileFormat.str || fileFormat == PdfFileFormat.mag) { + // pdfFileName.contains("mgc.pdf") || + // pdfFileName.contains("str.pdf") || + // pdfFileName.contains("mag.pdf") + /* ___________________________________________________________________________ */ + /*-------------------- load pdfs for mgc, str or mag ------------------------*/ + data_in = new DataInputStream(new BufferedInputStream(pdfStream)); + logger.debug("loadPdfs reading model of type " + fileFormat); + /* read vector size for spectrum */ + + // numStream = 1; // just one stream for mgc, str, mag. This is just to have only one + // type of pdf vector for all posible pdf's + // vsize = data_in.readInt(); + // vectorSize = vsize; + numMSDFlag = data_in.readInt(); + numStream = data_in.readInt(); + vectorSize = data_in.readInt(); + + vsize = vectorSize; + // System.out.println("loadPdfs: vsize = " + vsize); + + if (vsize < 0) + throw new MaryConfigurationException("loadPdfs: vector size of pdf must be positive."); + + /* Now we need the number of pdf's for each state */ + pdf = new double[numState][][][]; + numPdf = new int[numState]; + for (i = 0; i < numState; i++) { + numPdf[i] = data_in.readInt(); + logger.debug("loadPdfs: numPdf[state:" + i + "]=" + numPdf[i]); + if (numPdf[i] < 0) + throw new MaryConfigurationException("loadPdfs: #pdf at state " + i + " must be positive value."); + // System.out.println("nmceppdf[" + i + "] = " + nmceppdf[i]); + /* Now i know the size of mceppdf[#states][#leaves][vectorsize] */ + /* so i can allocate memory for mceppdf[][][] */ + pdf[i] = new double[numPdf[i]][numStream][2 * vsize]; + } + + /* read pdfs (mean, variance). (2*vsize because mean and diag variance */ + /* are allocated in only one vector. */ + for (i = 0; i < numState; i++) { + for (j = 0; j < numPdf[i]; j++) { + /* + * for( k=0; k<(2*vsize); k++ ){ pdf[i][j][0][k] = data_in.readFloat(); // [0] corresponds to stream, in this + * case just one. //System.out.println("pdf["+ i + "][" + j + "][0][" + k + "] =" + pdf[i][j][0][k]); } + */ + // NOTE: Here (hts_engine v1.04) the order is different as before, here mean and variance are saved + // consecutively + // so now the pdf contains: mean[0], vari[0], mean[1], vari[1], etc... + for (k = 0; k < vsize; k++) { + pdf[i][j][0][k] = data_in.readFloat(); // [0] corresponds to stream, in this case just one. + // System.out.println("pdf["+ i + "][" + j + "][0][" + k + "] =" + pdf[i][j][0][k]); + pdf[i][j][0][k + vsize] = data_in.readFloat(); + } + } + } + data_in.close(); + data_in = null; + + } + + return pdf; + + } /* method loadPdfs */ + + public static void main(String[] args) throws IOException, InterruptedException { + /* configure log info */ + org.apache.log4j.BasicConfigurator.configure(); + + String contextFile = "/project/mary/marcela/openmary/lib/voices/hsmm-slt/cmu_us_arctic_slt_a0001.pfeats"; + Scanner context = new Scanner(new BufferedReader(new FileReader(contextFile))); + String strContext = ""; + while (context.hasNext()) { + strContext += context.nextLine(); + strContext += "\n"; + } + context.close(); + // System.out.println(strContext); + FeatureDefinition feaDef = new FeatureDefinition(new BufferedReader(new StringReader(strContext)), false); + + CART[] mgcTree = null; + int numStates = 5; + String trickyPhones = "/project/mary/marcela/openmary/lib/voices/hsmm-slt/trickyPhones.txt"; + String treefile = "/project/mary/marcela/openmary/lib/voices/hsmm-slt/tree-dur.inf"; + String pdffile = "/project/mary/marcela/openmary/lib/voices/hsmm-slt/dur.pdf"; + int vSize; + + // Check if there are tricky phones, and create a PhoneTranslator object + PhoneTranslator phTranslator = new PhoneTranslator(new FileInputStream(trickyPhones)); + + HTSCARTReader htsReader = new HTSCARTReader(); + try { + mgcTree = htsReader.load(numStates, new FileInputStream(treefile), new FileInputStream(pdffile), PdfFileFormat.dur, + feaDef, phTranslator); + vSize = htsReader.getVectorSize(); + System.out.println("loaded " + pdffile + " vector size=" + vSize); + } catch (Exception e) { + System.out.println(e.getMessage()); + } + + } + +} diff --git a/marytts-unitselection/src/main/java/marytts/cart/io/MaryCARTReader.java b/marytts-unitselection/src/main/java/marytts/cart/io/MaryCARTReader.java new file mode 100644 index 0000000000..07ca4063d0 --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/cart/io/MaryCARTReader.java @@ -0,0 +1,402 @@ +/** + * Copyright 2006 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.cart.io; + +import java.io.BufferedInputStream; +import java.io.ByteArrayInputStream; +import java.io.DataInput; +import java.io.DataInputStream; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.util.Properties; + +import marytts.cart.CART; +import marytts.cart.DecisionNode; +import marytts.cart.LeafNode; +import marytts.cart.Node; +import marytts.exceptions.MaryConfigurationException; +import marytts.features.FeatureDefinition; +import marytts.util.data.MaryHeader; + +/** + * IO functions for CARTs in MaryCART format + * + * @author Marcela Charfuelan + */ +public class MaryCARTReader { + /** + * Load the cart from the given file + * + * @param fileName + * the file to load the cart from + * @param featDefinition + * the feature definition + * @param dummy + * unused, just here for compatibility with the FeatureFileIndexer. + * @throws IOException + * if a problem occurs while loading + */ + public CART load(String fileName) throws IOException, MaryConfigurationException { + FileInputStream fis = new FileInputStream(fileName); + try { + return loadFromStream(fis); + } finally { + fis.close(); + } + } + + /** + * Load the cart from the given file + * + * @param inStream + * the stream to load the cart from + * @param featDefinition + * the feature definition + * @param dummy + * unused, just here for compatibility with the FeatureFileIndexer. + * @throws IOException + * if a problem occurs while loading + */ + public CART loadFromStream(InputStream inStream) throws IOException, MaryConfigurationException { + // open the CART-File and read the header + DataInput raf = new DataInputStream(new BufferedInputStream(inStream)); + + MaryHeader maryHeader = new MaryHeader(raf); + if (!maryHeader.hasCurrentVersion()) { + throw new IOException("Wrong version of database file"); + } + if (maryHeader.getType() != MaryHeader.CARTS) { + throw new IOException("No CARTs file"); + } + + // Read properties + short propDataLength = raf.readShort(); + Properties props; + if (propDataLength == 0) { + props = null; + } else { + byte[] propsData = new byte[propDataLength]; + raf.readFully(propsData); + ByteArrayInputStream bais = new ByteArrayInputStream(propsData); + props = new Properties(); + props.load(bais); + bais.close(); + } + + // Read the feature definition + FeatureDefinition featureDefinition = new FeatureDefinition(raf); + + // read the decision nodes + int numDecNodes = raf.readInt(); // number of decision nodes + + // First we need to read all nodes into memory, then we can link them properly + // in terms of parent/child. + DecisionNode[] dns = new DecisionNode[numDecNodes]; + int[][] childIndexes = new int[numDecNodes][]; + for (int i = 0; i < numDecNodes; i++) { + // read one decision node + int featureNameIndex = raf.readInt(); + int nodeTypeNr = raf.readInt(); + DecisionNode.Type nodeType = DecisionNode.Type.values()[nodeTypeNr]; + int numChildren = 2; // for binary nodes + switch (nodeType) { + case BinaryByteDecisionNode: + int criterion = raf.readInt(); + dns[i] = new DecisionNode.BinaryByteDecisionNode(featureNameIndex, (byte) criterion, featureDefinition); + break; + case BinaryShortDecisionNode: + criterion = raf.readInt(); + dns[i] = new DecisionNode.BinaryShortDecisionNode(featureNameIndex, (short) criterion, featureDefinition); + break; + case BinaryFloatDecisionNode: + float floatCriterion = raf.readFloat(); + dns[i] = new DecisionNode.BinaryFloatDecisionNode(featureNameIndex, floatCriterion, featureDefinition); + break; + case ByteDecisionNode: + numChildren = raf.readInt(); + if (featureDefinition.getNumberOfValues(featureNameIndex) != numChildren) { + throw new IOException("Inconsistent cart file: feature " + featureDefinition.getFeatureName(featureNameIndex) + + " should have " + featureDefinition.getNumberOfValues(featureNameIndex) + + " values, but decision node " + i + " has only " + numChildren + " child nodes"); + } + dns[i] = new DecisionNode.ByteDecisionNode(featureNameIndex, numChildren, featureDefinition); + break; + case ShortDecisionNode: + numChildren = raf.readInt(); + if (featureDefinition.getNumberOfValues(featureNameIndex) != numChildren) { + throw new IOException("Inconsistent cart file: feature " + featureDefinition.getFeatureName(featureNameIndex) + + " should have " + featureDefinition.getNumberOfValues(featureNameIndex) + + " values, but decision node " + i + " has only " + numChildren + " child nodes"); + } + dns[i] = new DecisionNode.ShortDecisionNode(featureNameIndex, numChildren, featureDefinition); + } + // now read the children, indexes only: + childIndexes[i] = new int[numChildren]; + for (int k = 0; k < numChildren; k++) { + childIndexes[i][k] = raf.readInt(); + } + } + + // read the leaves + int numLeafNodes = raf.readInt(); // number of leaves, it does not include empty leaves + LeafNode[] lns = new LeafNode[numLeafNodes]; + + for (int j = 0; j < numLeafNodes; j++) { + // read one leaf node + int leafTypeNr = raf.readInt(); + LeafNode.LeafType leafNodeType = LeafNode.LeafType.values()[leafTypeNr]; + switch (leafNodeType) { + case IntArrayLeafNode: + int numData = raf.readInt(); + int[] data = new int[numData]; + for (int d = 0; d < numData; d++) { + data[d] = raf.readInt(); + } + lns[j] = new LeafNode.IntArrayLeafNode(data); + break; + case FloatLeafNode: + float stddev = raf.readFloat(); + float mean = raf.readFloat(); + lns[j] = new LeafNode.FloatLeafNode(new float[] { stddev, mean }); + break; + case IntAndFloatArrayLeafNode: + case StringAndFloatLeafNode: + int numPairs = raf.readInt(); + int[] ints = new int[numPairs]; + float[] floats = new float[numPairs]; + for (int d = 0; d < numPairs; d++) { + ints[d] = raf.readInt(); + floats[d] = raf.readFloat(); + } + if (leafNodeType == LeafNode.LeafType.IntAndFloatArrayLeafNode) + lns[j] = new LeafNode.IntAndFloatArrayLeafNode(ints, floats); + else + lns[j] = new LeafNode.StringAndFloatLeafNode(ints, floats); + break; + case FeatureVectorLeafNode: + throw new IllegalArgumentException("Reading feature vector leaf nodes is not yet implemented"); + case PdfLeafNode: + throw new IllegalArgumentException("Reading pdf leaf nodes is not yet implemented"); + } + } + + // Now, link up the decision nodes with their daughters + for (int i = 0; i < numDecNodes; i++) { + for (int k = 0; k < childIndexes[i].length; k++) { + int childIndex = childIndexes[i][k]; + if (childIndex < 0) { // a decision node + assert -childIndex - 1 < numDecNodes; + dns[i].addDaughter(dns[-childIndex - 1]); + } else if (childIndex > 0) { // a leaf node + dns[i].addDaughter(lns[childIndex - 1]); + } else { // == 0, an empty leaf + dns[i].addDaughter(null); + } + } + } + + Node rootNode; + if (dns.length > 0) { + rootNode = dns[0]; + // Now count all data once, so that getNumberOfData() + // will return the correct figure. + ((DecisionNode) rootNode).countData(); + } else if (lns.length > 0) { + rootNode = lns[0]; // single-leaf tree... + } else { + rootNode = null; + } + + // set the rootNode as the rootNode of cart + return new CART(rootNode, featureDefinition, props); + } + + /** + * Load the cart from the given file + * + * @param fileName + * the file to load the cart from + * @param featDefinition + * the feature definition + * @param dummy + * unused, just here for compatibility with the FeatureFileIndexer. + * @throws IOException + * if a problem occurs while loading + */ + private CART loadFromByteBuffer(String fileName) throws IOException, MaryConfigurationException { + // open the CART-File and read the header + FileInputStream fis = new FileInputStream(fileName); + FileChannel fc = fis.getChannel(); + ByteBuffer bb = fc.map(FileChannel.MapMode.READ_ONLY, 0, fc.size()); + fis.close(); + + MaryHeader maryHeader = new MaryHeader(bb); + if (!maryHeader.hasCurrentVersion()) { + throw new IOException("Wrong version of database file"); + } + if (maryHeader.getType() != MaryHeader.CARTS) { + throw new IOException("No CARTs file"); + } + + // Read properties + short propDataLength = bb.getShort(); + Properties props; + if (propDataLength == 0) { + props = null; + } else { + byte[] propsData = new byte[propDataLength]; + bb.get(propsData); + ByteArrayInputStream bais = new ByteArrayInputStream(propsData); + props = new Properties(); + props.load(bais); + bais.close(); + } + + // Read the feature definition + FeatureDefinition featureDefinition = new FeatureDefinition(bb); + + // read the decision nodes + int numDecNodes = bb.getInt(); // number of decision nodes + + // First we need to read all nodes into memory, then we can link them properly + // in terms of parent/child. + DecisionNode[] dns = new DecisionNode[numDecNodes]; + int[][] childIndexes = new int[numDecNodes][]; + for (int i = 0; i < numDecNodes; i++) { + // read one decision node + int featureNameIndex = bb.getInt(); + int nodeTypeNr = bb.getInt(); + DecisionNode.Type nodeType = DecisionNode.Type.values()[nodeTypeNr]; + int numChildren = 2; // for binary nodes + switch (nodeType) { + case BinaryByteDecisionNode: + int criterion = bb.getInt(); + dns[i] = new DecisionNode.BinaryByteDecisionNode(featureNameIndex, (byte) criterion, featureDefinition); + break; + case BinaryShortDecisionNode: + criterion = bb.getInt(); + dns[i] = new DecisionNode.BinaryShortDecisionNode(featureNameIndex, (short) criterion, featureDefinition); + break; + case BinaryFloatDecisionNode: + float floatCriterion = bb.getFloat(); + dns[i] = new DecisionNode.BinaryFloatDecisionNode(featureNameIndex, floatCriterion, featureDefinition); + break; + case ByteDecisionNode: + numChildren = bb.getInt(); + if (featureDefinition.getNumberOfValues(featureNameIndex) != numChildren) { + throw new IOException("Inconsistent cart file: feature " + featureDefinition.getFeatureName(featureNameIndex) + + " should have " + featureDefinition.getNumberOfValues(featureNameIndex) + + " values, but decision node " + i + " has only " + numChildren + " child nodes"); + } + dns[i] = new DecisionNode.ByteDecisionNode(featureNameIndex, numChildren, featureDefinition); + break; + case ShortDecisionNode: + numChildren = bb.getInt(); + if (featureDefinition.getNumberOfValues(featureNameIndex) != numChildren) { + throw new IOException("Inconsistent cart file: feature " + featureDefinition.getFeatureName(featureNameIndex) + + " should have " + featureDefinition.getNumberOfValues(featureNameIndex) + + " values, but decision node " + i + " has only " + numChildren + " child nodes"); + } + dns[i] = new DecisionNode.ShortDecisionNode(featureNameIndex, numChildren, featureDefinition); + } + // now read the children, indexes only: + childIndexes[i] = new int[numChildren]; + for (int k = 0; k < numChildren; k++) { + childIndexes[i][k] = bb.getInt(); + } + } + + // read the leaves + int numLeafNodes = bb.getInt(); // number of leaves, it does not include empty leaves + LeafNode[] lns = new LeafNode[numLeafNodes]; + + for (int j = 0; j < numLeafNodes; j++) { + // read one leaf node + int leafTypeNr = bb.getInt(); + LeafNode.LeafType leafNodeType = LeafNode.LeafType.values()[leafTypeNr]; + switch (leafNodeType) { + case IntArrayLeafNode: + int numData = bb.getInt(); + int[] data = new int[numData]; + for (int d = 0; d < numData; d++) { + data[d] = bb.getInt(); + } + lns[j] = new LeafNode.IntArrayLeafNode(data); + break; + case FloatLeafNode: + float stddev = bb.getFloat(); + float mean = bb.getFloat(); + lns[j] = new LeafNode.FloatLeafNode(new float[] { stddev, mean }); + break; + case IntAndFloatArrayLeafNode: + case StringAndFloatLeafNode: + int numPairs = bb.getInt(); + int[] ints = new int[numPairs]; + float[] floats = new float[numPairs]; + for (int d = 0; d < numPairs; d++) { + ints[d] = bb.getInt(); + floats[d] = bb.getFloat(); + } + if (leafNodeType == LeafNode.LeafType.IntAndFloatArrayLeafNode) + lns[j] = new LeafNode.IntAndFloatArrayLeafNode(ints, floats); + else + lns[j] = new LeafNode.StringAndFloatLeafNode(ints, floats); + break; + case FeatureVectorLeafNode: + throw new IllegalArgumentException("Reading feature vector leaf nodes is not yet implemented"); + case PdfLeafNode: + throw new IllegalArgumentException("Reading pdf leaf nodes is not yet implemented"); + } + } + + // Now, link up the decision nodes with their daughters + for (int i = 0; i < numDecNodes; i++) { + for (int k = 0; k < childIndexes[i].length; k++) { + int childIndex = childIndexes[i][k]; + if (childIndex < 0) { // a decision node + assert -childIndex - 1 < numDecNodes; + dns[i].addDaughter(dns[-childIndex - 1]); + } else if (childIndex > 0) { // a leaf node + dns[i].addDaughter(lns[childIndex - 1]); + } else { // == 0, an empty leaf + dns[i].addDaughter(null); + } + } + } + + Node rootNode; + if (dns.length > 0) { + rootNode = dns[0]; + // Now count all data once, so that getNumberOfData() + // will return the correct figure. + ((DecisionNode) rootNode).countData(); + } else if (lns.length > 0) { + rootNode = lns[0]; // single-leaf tree... + } else { + rootNode = null; + } + + // set the rootNode as the rootNode of cart + return new CART(rootNode, featureDefinition, props); + } +} diff --git a/marytts-unitselection/src/main/java/marytts/cart/io/MaryCARTWriter.java b/marytts-unitselection/src/main/java/marytts/cart/io/MaryCARTWriter.java new file mode 100644 index 0000000000..2216737708 --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/cart/io/MaryCARTWriter.java @@ -0,0 +1,345 @@ +/** + * Copyright 2006 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.cart.io; + +import java.io.BufferedOutputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataOutput; +import java.io.DataOutputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.PrintWriter; +import java.util.Properties; + +import marytts.cart.CART; +import marytts.cart.DecisionNode; +import marytts.cart.LeafNode; +import marytts.cart.Node; +import marytts.cart.DecisionNode.BinaryByteDecisionNode; +import marytts.cart.DecisionNode.BinaryFloatDecisionNode; +import marytts.cart.DecisionNode.BinaryShortDecisionNode; +import marytts.cart.LeafNode.FeatureVectorLeafNode; +import marytts.cart.LeafNode.FloatLeafNode; +import marytts.cart.LeafNode.IntAndFloatArrayLeafNode; +import marytts.cart.LeafNode.IntArrayLeafNode; +import marytts.cart.LeafNode.LeafType; +import marytts.features.FeatureVector; +import marytts.util.MaryUtils; +import marytts.util.data.MaryHeader; + +import org.apache.log4j.Logger; + +/** + * IO functions for CARTs in MaryCART format + * + * @author Marcela Charfuelan + */ +public class MaryCARTWriter { + + protected Logger logger = MaryUtils.getLogger(this.getClass().getName()); + + /** + * Dump the CARTs in MaryCART format + * + * @param destDir + * the destination directory + */ + public void dumpMaryCART(CART cart, String destFile) throws IOException { + if (cart == null) + throw new NullPointerException("Cannot dump null CART"); + if (destFile == null) + throw new NullPointerException("No destination file"); + + logger.debug("Dumping CART in MaryCART format to " + destFile + " ..."); + + // Open the destination file (cart.bin) and output the header + DataOutputStream out = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(destFile))); + // create new CART-header and write it to output file + MaryHeader hdr = new MaryHeader(MaryHeader.CARTS); + hdr.writeTo(out); + + Properties props = cart.getProperties(); + if (props == null) { + out.writeShort(0); + } else { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + props.store(baos, null); + byte[] propData = baos.toByteArray(); + out.writeShort(propData.length); + out.write(propData); + } + + // feature definition + cart.getFeatureDefinition().writeBinaryTo(out); + + // dump CART + dumpBinary(cart.getRootNode(), out); + + // finish + out.close(); + logger.debug(" ... done\n"); + } + + public void toTextOut(CART cart, PrintWriter pw) throws IOException { + try { + int id[] = new int[2]; + id[0] = 0; // number of decision nodes + id[1] = 0; // number of leaf nodes + + // System.out.println("Total number of nodes:" + rootNode.getNumberOfNodes()); + setUniqueNodeId(cart.getRootNode(), id); + pw.println("Num decision nodes= " + id[0] + " Num leaf nodes= " + id[1]); + printDecisionNodes(cart.getRootNode(), null, pw); + pw.println("\n----------------\n"); + printLeafNodes(cart.getRootNode(), null, pw); + + pw.flush(); + pw.close(); + } catch (IOException ioe) { + IOException newIOE = new IOException("Error dumping CART to standard output"); + newIOE.initCause(ioe); + throw newIOE; + } + } + + private void setUniqueNodeId(Node node, int id[]) throws IOException { + + int thisIdNode; + String leafstr = ""; + + // if the node is decision node + if (node.getNumberOfNodes() > 1) { + assert node instanceof DecisionNode; + DecisionNode decNode = (DecisionNode) node; + id[0]--; + decNode.setUniqueDecisionNodeId(id[0]); + String strNode = ""; + // this.decNodeStr = ""; + thisIdNode = id[0]; + + // add Ids to the daughters + for (int i = 0; i < decNode.getNumberOfDaugthers(); i++) { + setUniqueNodeId(decNode.getDaughter(i), id); + } + + } else { // the node is a leaf node + assert node instanceof LeafNode; + LeafNode leaf = (LeafNode) node; + if (leaf.isEmpty()) { + leaf.setUniqueLeafId(0); + } else { + id[1]++; + leaf.setUniqueLeafId(id[1]); + } + + } + + } + + private void dumpBinary(Node rootNode, DataOutput os) throws IOException { + try { + + int id[] = new int[2]; + id[0] = 0; // number of decision nodes + id[1] = 0; // number of leaf nodes + // first add unique identifiers to decision nodes and leaf nodes + setUniqueNodeId(rootNode, id); + + // write the number of decision nodes + os.writeInt(Math.abs(id[0])); + // lines that start with a negative number are decision nodes + printDecisionNodes(rootNode, os, null); + + // write the number of leaves. + os.writeInt(id[1]); + // lines that start with id are leaf nodes + printLeafNodes(rootNode, (DataOutputStream) os, null); + + } catch (IOException ioe) { + IOException newIOE = new IOException("Error dumping CART to output stream"); + newIOE.initCause(ioe); + throw newIOE; + } + } + + private void printDecisionNodes(Node node, DataOutput out, PrintWriter pw) throws IOException { + if (!(node instanceof DecisionNode)) + return; // nothing to do here + + DecisionNode decNode = (DecisionNode) node; + int id = decNode.getUniqueDecisionNodeId(); + String nodeDefinition = decNode.getNodeDefinition(); + int featureIndex = decNode.getFeatureIndex(); + DecisionNode.Type nodeType = decNode.getDecisionNodeType(); + + if (out != null) { + // dump in binary form to output + out.writeInt(featureIndex); + out.writeInt(nodeType.ordinal()); + // Now, questionValue, which depends on nodeType + switch (nodeType) { + case BinaryByteDecisionNode: + out.writeInt(((BinaryByteDecisionNode) decNode).getCriterionValueAsByte()); + assert decNode.getNumberOfDaugthers() == 2; + break; + case BinaryShortDecisionNode: + out.writeInt(((BinaryShortDecisionNode) decNode).getCriterionValueAsShort()); + assert decNode.getNumberOfDaugthers() == 2; + break; + case BinaryFloatDecisionNode: + out.writeFloat(((BinaryFloatDecisionNode) decNode).getCriterionValueAsFloat()); + assert decNode.getNumberOfDaugthers() == 2; + break; + case ByteDecisionNode: + case ShortDecisionNode: + out.writeInt(decNode.getNumberOfDaugthers()); + } + + // The child nodes + for (int i = 0, n = decNode.getNumberOfDaugthers(); i < n; i++) { + Node daughter = decNode.getDaughter(i); + if (daughter instanceof DecisionNode) { + out.writeInt(((DecisionNode) daughter).getUniqueDecisionNodeId()); + } else { + assert daughter instanceof LeafNode; + out.writeInt(((LeafNode) daughter).getUniqueLeafId()); + } + } + } + if (pw != null) { + // dump to print writer + StringBuilder strNode = new StringBuilder(id + " " + nodeDefinition); + for (int i = 0, n = decNode.getNumberOfDaugthers(); i < n; i++) { + strNode.append(" "); + Node daughter = decNode.getDaughter(i); + if (daughter instanceof DecisionNode) { + strNode.append(((DecisionNode) daughter).getUniqueDecisionNodeId()); + } else { + assert daughter instanceof LeafNode; + strNode.append("id").append(((LeafNode) daughter).getUniqueLeafId()); + } + } + pw.println(strNode.toString()); + } + // add the daughters + for (int i = 0; i < ((DecisionNode) node).getNumberOfDaugthers(); i++) { + if (((DecisionNode) node).getDaughter(i).getNumberOfNodes() > 1) + printDecisionNodes(((DecisionNode) node).getDaughter(i), out, pw); + } + } + + /** This function will print the leaf nodes only, but it goes through all the decision nodes. */ + private void printLeafNodes(Node node, DataOutput out, PrintWriter pw) throws IOException { + // If the node does not have leaves then it just return. + // I we are in a decision node then print the leaves of the daughters. + Node nextNode; + if (node.getNumberOfNodes() > 1) { + assert node instanceof DecisionNode; + DecisionNode decNode = (DecisionNode) node; + for (int i = 0; i < decNode.getNumberOfDaugthers(); i++) { + nextNode = decNode.getDaughter(i); + printLeafNodes(nextNode, out, pw); + } + } else { + assert node instanceof LeafNode; + LeafNode leaf = (LeafNode) node; + if (leaf.getUniqueLeafId() == 0) // empty leaf, do not write + return; + LeafType leafType = leaf.getLeafNodeType(); + if (leafType == LeafType.FeatureVectorLeafNode) { + leafType = LeafType.IntArrayLeafNode; + // save feature vector leaf nodes as int array leaf nodes + } + if (out != null) { + // Leaf node type + out.writeInt(leafType.ordinal()); + } + if (pw != null) { + pw.print("id" + leaf.getUniqueLeafId() + " " + leafType); + } + switch (leaf.getLeafNodeType()) { + case IntArrayLeafNode: + int data[] = ((IntArrayLeafNode) leaf).getIntData(); + // Number of data points following: + if (out != null) + out.writeInt(data.length); + if (pw != null) + pw.print(" " + data.length); + // for each index, write the index + for (int i = 0; i < data.length; i++) { + if (out != null) + out.writeInt(data[i]); + if (pw != null) + pw.print(" " + data[i]); + } + break; + case FloatLeafNode: + float stddev = ((FloatLeafNode) leaf).getStDeviation(); + float mean = ((FloatLeafNode) leaf).getMean(); + if (out != null) { + out.writeFloat(stddev); + out.writeFloat(mean); + } + if (pw != null) { + pw.print(" 1 " + stddev + " " + mean); + } + break; + case IntAndFloatArrayLeafNode: + case StringAndFloatLeafNode: + int data1[] = ((IntAndFloatArrayLeafNode) leaf).getIntData(); + float floats[] = ((IntAndFloatArrayLeafNode) leaf).getFloatData(); + // Number of data points following: + if (out != null) + out.writeInt(data1.length); + if (pw != null) + pw.print(" " + data1.length); + // for each index, write the index and then its float + for (int i = 0; i < data1.length; i++) { + if (out != null) { + out.writeInt(data1[i]); + out.writeFloat(floats[i]); + } + if (pw != null) + pw.print(" " + data1[i] + " " + floats[i]); + } + break; + case FeatureVectorLeafNode: + FeatureVector fv[] = ((FeatureVectorLeafNode) leaf).getFeatureVectors(); + // Number of data points following: + if (out != null) + out.writeInt(fv.length); + if (pw != null) + pw.print(" " + fv.length); + // for each feature vector, write the index + for (int i = 0; i < fv.length; i++) { + if (out != null) + out.writeInt(fv[i].getUnitIndex()); + if (pw != null) + pw.print(" " + fv[i].getUnitIndex()); + } + break; + case PdfLeafNode: + throw new IllegalArgumentException("Writing of pdf leaf nodes not yet implemented"); + } + if (pw != null) + pw.println(); + } + } +} diff --git a/marytts-unitselection/src/main/java/marytts/cart/io/WagonCARTReader.java b/marytts-unitselection/src/main/java/marytts/cart/io/WagonCARTReader.java new file mode 100644 index 0000000000..17076fbcd7 --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/cart/io/WagonCARTReader.java @@ -0,0 +1,607 @@ +/** + * Copyright 2006 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.cart.io; + +import java.io.BufferedInputStream; +import java.io.BufferedReader; +import java.io.DataInput; +import java.io.DataInputStream; +import java.io.EOFException; +import java.io.FileInputStream; +import java.io.IOException; +import java.util.StringTokenizer; + +import marytts.cart.DecisionNode; +import marytts.cart.LeafNode; +import marytts.cart.Node; +import marytts.exceptions.MaryConfigurationException; +import marytts.features.FeatureDefinition; +import marytts.features.FeatureVector; +import marytts.util.data.MaryHeader; + +/** + * IO functions for CARTs in WagonCART format + * + * @author Anna Hunecke, Marc Schröder, Marcela Charfuelan + */ +public class WagonCARTReader { + + private Node rootNode; + private Node lastNode; + + // knows the index numbers and types of the features used in DecisionNodes + private FeatureDefinition featDef; + + private int openBrackets; + + // Since it is not known from the wagon file lines which kind of leaves + // should be read, a leafType argument should be provided when creating + // this class. + private LeafNode.LeafType leafType; + + // added because StringCART + private int targetFeature; + + /** + * When creating a WagonCARTReader provide a tree type + * + * @param treeType + * ClasificationTree, ExtendedClassificationTree, RegressionTree, or TopLevelTree. + * + *

+ * ClasificationTree --> IntArrayLeafNode + *

+ * ExtendedClassificationTree --> IntAndFloatArrayLeafNode + *

+ * RegressionTree --> FloatLeafNode + *

+ * TopLevelTree --> FeatureVectorLeafNode + *

+ * StringCART --> StringAndFloatLeafNode + */ + public WagonCARTReader(LeafNode.LeafType leafType) { + this.leafType = leafType; + } + + /** + * For a line representing a leaf in Wagon format, create a leaf. This method decides which implementation of LeafNode is + * used, i.e. which data format is appropriate. Lines are of the form (( )...( )) 0)) + * + * @param line + * a line from a wagon cart file, representing a leaf + * @return a leaf node representing the line. + */ + protected LeafNode createLeafNode(String line) { + if (leafType == LeafNode.LeafType.IntArrayLeafNode) + return (createIntArrayLeafNode(line)); + else if (leafType == LeafNode.LeafType.IntAndFloatArrayLeafNode) + return (createIntAndFloatArrayLeafNode(line)); + else if (leafType == LeafNode.LeafType.FloatLeafNode) + return (createFloatLeafNode(line)); + else if (leafType == LeafNode.LeafType.FeatureVectorLeafNode) + return (createFeatureVectorLeafNode(line)); + else if (leafType == LeafNode.LeafType.StringAndFloatLeafNode) + return (createStringAndFloatLeafNode(line)); + else + return null; + } + + // in case of using the reader more than once for different root nodes. + private void cleadReader() { + rootNode = null; + lastNode = null; + featDef = null; + openBrackets = 0; + } + + /** + * + * This loads a cart from a wagon tree in textual format, from a reader. + * + * @param reader + * the Reader providing the wagon tree + * @param featDefinition + * @throws IOException + */ + public Node load(BufferedReader reader, FeatureDefinition featDefinition) throws IOException { + cleadReader(); + featDef = featDefinition; + openBrackets = 0; + String line = reader.readLine(); + if (line.equals("")) {// first line is empty, read again + line = reader.readLine(); + } + // each line corresponds to a node + // for each line + while (line != null) { + if (!line.startsWith(";;") && !line.equals("")) { + // parse the line and add the node + + parseAndAdd(line); + } + line = reader.readLine(); + } + // make sure we closed as many brackets as we opened + if (openBrackets != 0) { + throw new IOException("Error loading CART: bracket mismatch"); + } + // Now count all data once, so that getNumberOfData() + // will return the correct figure. + if (rootNode instanceof DecisionNode) + ((DecisionNode) rootNode).countData(); + + return rootNode; + } + + /** + * Load the cart from the given file + * + * @param fileName + * the file to load the cart from + * @param featDefinition + * the feature definition + * @param dummy + * unused, just here for compatibility with the FeatureFileIndexer. + * @throws IOException + * if a problem occurs while loading + */ + // TODO: CHECK! do we need that String[] dummy??? + public Node load(String fileName, FeatureDefinition featDefinition, String[] dummy) throws IOException, + MaryConfigurationException { + cleadReader(); + // System.out.println("Loading file"); + // open the CART-File and read the header + DataInput raf = new DataInputStream(new BufferedInputStream(new FileInputStream(fileName))); + MaryHeader maryHeader = new MaryHeader(raf); + if (!maryHeader.hasCurrentVersion()) { + throw new IOException("Wrong version of database file"); + } + if (maryHeader.getType() != MaryHeader.CARTS) { + throw new IOException("No CARTs file"); + } + // System.out.println("Reading CART"); + // discard number of CARTs and CART name + // TODO: Change format of CART-File + int numNodes = raf.readInt(); + raf.readUTF(); + + // load the CART + featDef = featDefinition; + // get the backtrace information + openBrackets = 0; + // Not elegant, but robust + try { + while (true) { + // parse the line and add the node + int length = raf.readInt(); + char[] cartChars = new char[length]; + for (int i = 0; i < length; i++) { + cartChars[i] = raf.readChar(); + } + String cart = new String(cartChars); + // System.out.println(cart); + parseAndAdd(cart); + } + } catch (EOFException eof) { + } + + // make sure we closed as many brackets as we opened + if (openBrackets != 0) { + throw new IOException("Error loading CART: bracket mismatch: " + openBrackets); + } + // Now count all data once, so that getNumberOfData() + // will return the correct figure. + if (rootNode instanceof DecisionNode) + ((DecisionNode) rootNode).countData(); + // System.out.println("Done"); + + return rootNode; + } + + /** + * Creates a node from the given input line and add it to the CART. + * + * @param line + * a line of input to parse + * @throws IOException + * if the line has an unexpected format + */ + private void parseAndAdd(String line) throws IOException { + // remove whitespace + line = line.trim(); + // at beginning of String there should be at least two opening brackets + if (!(line.startsWith("(("))) { + throw new IOException("Invalid input line for CART: " + line); + } + if (Character.isLetter(line.charAt(2)) && !line.substring(2, 6).equals("nan ")) { // we have a node + openBrackets++; // do not count first bracket + + // get the properties of the node + StringTokenizer tokenizer = new StringTokenizer(line, " "); + String feature = tokenizer.nextToken().substring(2); + String type = tokenizer.nextToken(); + String value = tokenizer.nextToken(); + value = value.substring(0, value.length() - 1); + // some values are enclosed in double quotes: + if (value.startsWith("\"") && value.endsWith("\"") && value.length() > 2) + value = value.substring(1, value.length() - 1); + + // a literal double quote is escaped by backslash, so unescape it: + if (value.contains("\\\"")) { + value = value.replaceAll("\\\\\"", "\""); + } + + // build new node depending on type + + Node nextNode; + try { + if (type.equals("is")) { + if (featDef.isByteFeature(feature)) { + nextNode = new DecisionNode.BinaryByteDecisionNode(feature, value, featDef); + } else { + nextNode = new DecisionNode.BinaryShortDecisionNode(feature, value, featDef); + } + } else { + if (type.equals("<")) { + nextNode = new DecisionNode.BinaryFloatDecisionNode(feature, Float.parseFloat(value), featDef); + } else { + if (type.equals("isShortOf")) { + nextNode = new DecisionNode.ShortDecisionNode(feature, Integer.parseInt(value), featDef); + } else { + if (type.equals("isByteOf")) { + nextNode = new DecisionNode.ByteDecisionNode(feature, Integer.parseInt(value), featDef); + } else { + throw new IOException("Unknown node type : " + type); + } + } + } + } + + } catch (Exception exc) { + throw new RuntimeException("Cannot create decision node for cart line: '" + line + "'", exc); + } + + if (lastNode != null) { + // this node is the daughter of the last node + ((DecisionNode) lastNode).addDaughter(nextNode); + } else { + // this is the rootNode + rootNode = nextNode; + nextNode.setIsRoot(true); + } + + // go one step down + lastNode = nextNode; + + } else { // we have a leaf + + Node nextNode = createLeafNode(line); + + // set the relations of this node to the others + if (lastNode == null) { // this node is the root + rootNode = nextNode; + nextNode.setIsRoot(true); + } else { // this node is a daughter of lastNode + ((DecisionNode) lastNode).addDaughter(nextNode); + } + + // look at the bracketing at the end of the line: + // get the last token out of the tokenizer + StringTokenizer tokenizer = new StringTokenizer(line, " "); + for (int i = 0, numTokens = tokenizer.countTokens(); i < numTokens - 1; i++) { + tokenizer.nextToken(); + } + String lastToken = tokenizer.nextToken(); + + // lastToken should look like "0))" + // more than two brackets mean that this is + // the last daughter of one or more nodes + int length = lastToken.length(); + // start looking at the characters after "0))" + int index = lastToken.indexOf(')') + 2; + + while (index < length) { // while we have more characters + char nextChar = lastToken.charAt(index); + if (nextChar == ')') { + // if the next character is a closing bracket + openBrackets--; + // this is the last daughter of lastNode, + // try going one step up + if (lastNode.isRoot()) { + if (index + 1 != length) { + // lastNode should not be the root, + // unless we are at the last bracket + throw new IOException("Too many closing brackets in line " + line); + } + } else { // you can go one step up + nextNode = lastNode; + lastNode = lastNode.getMother(); + } + } else { + // nextChar is not a closing bracket; + // something went wrong here + throw new IOException("Expected closing bracket in line " + line + ", but found " + nextChar); + } + index++; + } + // for debugging + if (nextNode != null) { + int nodeIndex = nextNode.getNodeIndex(); + } + + } + } + + protected LeafNode createIntArrayLeafNode(String line) { + StringTokenizer tok = new StringTokenizer(line, " "); + // read the indices from the tokenized String + int numTokens = tok.countTokens(); + int index = 0; + // The data to be saved in the leaf node: + int[] indices; + if (numTokens == 2) { // we do not have any indices + // discard useless token + tok.nextToken(); + indices = new int[0]; + } else { + indices = new int[(numTokens - 1) / 2]; + + while (index * 2 < numTokens - 1) { // while we are not at the + // last token + String nextToken = tok.nextToken(); + if (index == 0) { + // we are at first token, discard all open brackets + nextToken = nextToken.substring(4); + } else { + // we are not at first token, only one open bracket + nextToken = nextToken.substring(1); + } + // store the index of the unit + indices[index] = Integer.parseInt(nextToken); + // discard next token + tok.nextToken(); + // increase index + index++; + } + } + return new LeafNode.IntArrayLeafNode(indices); + } + + protected LeafNode createIntAndFloatArrayLeafNode(String line) { + StringTokenizer tok = new StringTokenizer(line, " "); + // read the indices from the tokenized String + int numTokens = tok.countTokens(); + int index = 0; + // The data to be saved in the leaf node: + int[] indices; + // The floats to be saved in the leaf node: + float[] probs; + + // System.out.println("Line: "+line+", numTokens: "+numTokens); + + if (numTokens == 2) { // we do not have any indices + // discard useless token + tok.nextToken(); + indices = new int[0]; + probs = new float[0]; + } else { + indices = new int[(numTokens - 1) / 2]; + // same length + probs = new float[indices.length]; + + while (index * 2 < numTokens - 1) { + String token = tok.nextToken(); + if (index == 0) { + token = token.substring(4); + } else { + token = token.substring(1); + } + // System.out.println("int-token: "+token); + indices[index] = Integer.parseInt(token); + + token = tok.nextToken(); + int lastIndex = token.length() - 1; + if ((index * 2) == (numTokens - 3)) { + token = token.substring(0, lastIndex - 1); + if (token.equals("inf")) { + probs[index] = 10000; + index++; + continue; + } + if (token.equals("nan")) { + probs[index] = -1; + index++; + continue; + } + } else { + token = token.substring(0, lastIndex); + if (token.equals("inf")) { + probs[index] = 1000000; + index++; + continue; + } + if (token.equals("nan")) { + probs[index] = -1; + index++; + continue; + } + } + // System.out.println("float-token: "+token); + probs[index] = Float.parseFloat(token); + index++; + } // end while + + } // end if + + return new LeafNode.IntAndFloatArrayLeafNode(indices, probs); + } + + protected LeafNode createFloatLeafNode(String line) { + StringTokenizer tok = new StringTokenizer(line, " "); + // read the indices from the tokenized String + int numTokens = tok.countTokens(); + if (numTokens != 2) { // we need exactly one value pair + throw new IllegalArgumentException("Expected two tokens in line, got " + numTokens + ": '" + line + "'"); + } + + // The data to be saved in the leaf node: + float[] data = new float[2]; // stddev and mean; + String nextToken = tok.nextToken(); + nextToken = nextToken.substring(2); + try { + data[0] = Float.parseFloat(nextToken); + } catch (NumberFormatException nfe) { + data[0] = 0; // cannot make sense of the standard deviation + } + nextToken = tok.nextToken(); + nextToken = nextToken.substring(0, nextToken.indexOf(")")); + try { + data[1] = Float.parseFloat(nextToken); + } catch (NumberFormatException nfe) { + data[1] = 0; + } + return new LeafNode.FloatLeafNode(data); + } + + protected LeafNode createFeatureVectorLeafNode(String line) { + + StringTokenizer tok = new StringTokenizer(line, " "); + // read the indices from the tokenized String + int numTokens = tok.countTokens(); + int index = 0; + // The data to be saved in the leaf node: + if (numTokens != 2) { + // leaf is not empty -> error + throw new Error("Leaf in line " + line + " is not empty"); + } + // discard useless token + tok.nextToken(); + return new LeafNode.FeatureVectorLeafNode(); + + } + + /** + * Fill the FeatureVector leafs of a tree with the given feature vectors. This function is only used in TopLeavelTree. + * + * @param root + * node of the tree. + * @param featureVectors + * the feature vectors. + */ + public void fillLeafs(Node root, FeatureVector[] featureVectors) { + if (leafType == LeafNode.LeafType.FeatureVectorLeafNode) { + rootNode = root; + Node currentNode = rootNode; + Node prevNode = null; + + // loop trough the feature vectors + for (int i = 0; i < featureVectors.length; i++) { + currentNode = rootNode; + prevNode = null; + FeatureVector featureVector = featureVectors[i]; + // logger.debug("Starting cart at "+nodeIndex); + while (!(currentNode instanceof LeafNode)) { + // while we have not reached the bottom, + // get the next node based on the features of the target + prevNode = currentNode; + currentNode = ((DecisionNode) currentNode).getNextNode(featureVector); + // logger.debug(decision.toString() + " result '"+ + // decision.findFeature(item) + "' => "+ nodeIndex); + } + // add the feature vector to the leaf node + ((LeafNode.FeatureVectorLeafNode) currentNode).addFeatureVector(featureVector); + } + } else + throw new IllegalArgumentException("The leaves of this tree are not FeatureVectorLeafNode."); + + } + + protected LeafNode createStringAndFloatLeafNode(String line) { + // Note: this code is identical to createIntAndFloatArrayLeafNode(), + // except for the last line. + StringTokenizer tok = new StringTokenizer(line, " "); + // read the indices from the tokenized String + int numTokens = tok.countTokens(); + int index = 0; + // The data to be saved in the leaf node: + int[] indices; + // The floats to be saved in the leaf node: + float[] probs; + + // System.out.println("Line: "+line+", numTokens: "+numTokens); + + if (numTokens == 2) { // we do not have any indices + // discard useless token + tok.nextToken(); + indices = new int[0]; + probs = new float[0]; + } else { + indices = new int[(numTokens - 1) / 2]; + // same length + probs = new float[indices.length]; + + while (index * 2 < numTokens - 1) { + String token = tok.nextToken(); + if (index == 0) { + token = token.substring(4); + } else { + token = token.substring(1); + } + // System.out.println("int-token: "+token); + indices[index] = Integer.parseInt(token); + + token = tok.nextToken(); + int lastIndex = token.length() - 1; + if ((index * 2) == (numTokens - 3)) { + token = token.substring(0, lastIndex - 1); + if (token.equals("inf")) { + probs[index] = 10000; + index++; + continue; + } + if (token.equals("nan")) { + probs[index] = -1; + index++; + continue; + } + } else { + token = token.substring(0, lastIndex); + if (token.equals("inf")) { + probs[index] = 1000000; + index++; + continue; + } + if (token.equals("nan")) { + probs[index] = -1; + index++; + continue; + } + } + // System.out.println("float-token: "+token); + probs[index] = Float.parseFloat(token); + index++; + } // end while + + } // end if + + return new LeafNode.StringAndFloatLeafNode(indices, probs); + } + +} diff --git a/marytts-unitselection/src/main/java/marytts/cart/io/WagonCARTWriter.java b/marytts-unitselection/src/main/java/marytts/cart/io/WagonCARTWriter.java new file mode 100644 index 0000000000..d46204129b --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/cart/io/WagonCARTWriter.java @@ -0,0 +1,379 @@ +/** + * Copyright 2006 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.cart.io; + +import java.io.BufferedOutputStream; +import java.io.DataOutput; +import java.io.DataOutputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.PrintWriter; + +import marytts.cart.CART; +import marytts.cart.DecisionNode; +import marytts.cart.Node; +import marytts.cart.LeafNode.FeatureVectorLeafNode; +import marytts.cart.LeafNode.FloatLeafNode; +import marytts.cart.LeafNode.IntAndFloatArrayLeafNode; +import marytts.cart.LeafNode.IntArrayLeafNode; +import marytts.features.FeatureVector; +import marytts.util.data.MaryHeader; + +/** + * IO functions for CARTs in WagonCART format + * + * @author Anna Hunecke, Marc Schröder, Marcela Charfuelan + */ +public class WagonCARTWriter { + + /** + * Dump the CARTs in the cart map to destinationDir/CARTS.bin + * + * @param cart + * tree + * @param destDir + * the destination directory + */ + public void dumpWagonCART(CART cart, String destFile) throws IOException { + System.out.println("Dumping CART to " + destFile + " ..."); + + // Open the destination file (cart.bin) and output the header + DataOutputStream out = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(destFile))); + // create new CART-header and write it to output file + MaryHeader hdr = new MaryHeader(MaryHeader.CARTS); + hdr.writeTo(out); + + // write number of nodes + out.writeInt(cart.getNumNodes()); + String name = ""; + // dump name and CART + out.writeUTF(name); + // dump CART + dumpBinary(cart, out); + + // finish + out.close(); + System.out.println(" ... done\n"); + } + + /** + * Debug output to a text file + * + * @param cart + * the cart tree + * @param pw + * the print writer of the text file + * @throws IOException + */ + public void toTextOut(CART cart, PrintWriter pw) throws IOException { + try { + toWagonFormat(cart.getRootNode(), null, "", pw); + pw.flush(); + pw.close(); + } catch (IOException ioe) { + IOException newIOE = new IOException("Error dumping CART to standard output"); + newIOE.initCause(ioe); + throw newIOE; + } + } + + /** + * Dumps this CART to the output stream in WagonFormat. + * + * @param os + * the output stream + * @param cart + * the cart tree + * + * @throws IOException + * if an error occurs during output + */ + public void dumpBinary(CART cart, DataOutput os) throws IOException { + try { + toWagonFormat(cart.getRootNode(), (DataOutputStream) os, null, null); + } catch (IOException ioe) { + IOException newIOE = new IOException("Error dumping CART to output stream"); + newIOE.initCause(ioe); + throw newIOE; + } + } + + private void toWagonFormat(Node node, DataOutputStream out, String extension, PrintWriter pw) throws IOException { + + if (node instanceof DecisionNode) + toWagonFormat(((DecisionNode) node), out, extension, pw); + + else if (node instanceof FeatureVectorLeafNode) + toWagonFormat(((FeatureVectorLeafNode) node), out, extension, pw); + + else if (node instanceof FloatLeafNode) + toWagonFormat(((FloatLeafNode) node), out, extension, pw); + + // As StringAndFloatLeafNode is just a convenience wrapper around IntAndFloatArrayLeafNode, + // there is no need to distinguish them in IO. + else if (node instanceof IntAndFloatArrayLeafNode) + toWagonFormat(((IntAndFloatArrayLeafNode) node), out, extension, pw); + + else if (node instanceof IntArrayLeafNode) + toWagonFormat(((IntArrayLeafNode) node), out, extension, pw); + + } + + /** + * Writes the Cart to the given DataOut in Wagon Format + * + * @param out + * the outputStream + * @param extension + * the extension that is added to the last daughter + */ + private void toWagonFormat(DecisionNode node, DataOutputStream out, String extension, PrintWriter pw) throws IOException { + if (out != null) { + // dump to output stream + // two open brackets + definition of node + writeStringToOutput("((" + node.getNodeDefinition() + ")", out); + } else { + // dump to Standard out + // two open brackets + definition of node + // System.out.println("(("+getNodeDefinition()); + } + if (pw != null) { + // dump to print writer + // two open brackets + definition of node + pw.println("((" + node.getNodeDefinition() + ")"); + } + // add the daughters + for (int i = 0; i < node.getNumberOfDaugthers(); i++) { + + if (node.getDaughter(i) == null) { + String nullDaughter = ""; + + if (i + 1 != node.getNumberOfDaugthers()) { + nullDaughter = "((() 0))"; + + } else { + // extension must be added to last daughter + if (extension != null) { + nullDaughter = "((() 0)))" + extension; + + } else { + // we are in the root node, add a closing bracket + nullDaughter = "((() 0)))"; + } + } + + if (out != null) { + // dump to output stream + writeStringToOutput(nullDaughter, out); + } else { + // dump to Standard out + // System.out.println(nullDaughter); + } + if (pw != null) { + pw.println(" " + nullDaughter); + } + } else { + if (i + 1 != node.getNumberOfDaugthers()) { + + toWagonFormat(node.getDaughter(i), out, "", pw); + // daughters[i].toWagonFormat(out, "", pw); + } else { + + // extension must be added to last daughter + if (extension != null) { + toWagonFormat(node.getDaughter(i), out, ")" + extension, pw); + // daughters[i].toWagonFormat(out, ")" + extension, pw); + } else { + // we are in the root node, add a closing bracket + toWagonFormat(node.getDaughter(i), out, ")", pw); + // daughters[i].toWagonFormat(out, ")", pw); + } + } + } + } + } + + /** + * Writes the Cart to the given DataOut in Wagon Format + * + * @param out + * the outputStream + * @param extension + * the extension that is added to the last daughter + */ + private void toWagonFormat(FeatureVectorLeafNode node, DataOutputStream out, String extension, PrintWriter pw) + throws IOException { + StringBuilder sb = new StringBuilder(); + FeatureVector fv[] = node.getFeatureVectors(); + + // open three brackets + sb.append("((("); + // make sure that we have a feature vector array + // this is done when calling getFeatureVectors(). + // if (growable && + // (featureVectors == null + // || featureVectors.length == 0)){ + // featureVectors = (FeatureVector[]) + // featureVectorList.toArray( + // new FeatureVector[featureVectorList.size()]); + // } + // for each index, write the index and then a pseudo float + for (int i = 0; i < fv.length; i++) { + sb.append("(" + fv[i].getUnitIndex() + " 0)"); + if (i + 1 != fv.length) { + sb.append(" "); + } + } + // write the ending + sb.append(") 0))" + extension); + // dump the whole stuff + if (out != null) { + // write to output stream + + writeStringToOutput(sb.toString(), out); + } else { + // write to Standard out + // System.out.println(sb.toString()); + } + if (pw != null) { + // dump to printwriter + pw.println(sb.toString()); + } + } + + /** + * Writes the Cart to the given DataOut in Wagon Format + * + * @param out + * the outputStream + * @param extension + * the extension that is added to the last daughter + */ + private void toWagonFormat(FloatLeafNode node, DataOutputStream out, String extension, PrintWriter pw) throws IOException { + String s = "((" + node.getStDeviation() // stddev + + " " + node.getMean() // mean + + "))"; + // dump the whole stuff + if (out != null) { + // write to output stream + + writeStringToOutput(s, out); + } else { + // write to Standard out + // System.out.println(sb.toString()); + } + if (pw != null) { + // dump to printwriter + pw.println(s); + } + } + + /** + * Writes the Cart to the given DataOut in Wagon Format + * + * @param out + * the outputStream + * @param extension + * the extension that is added to the last daughter + */ + private void toWagonFormat(IntAndFloatArrayLeafNode node, DataOutputStream out, String extension, PrintWriter pw) + throws IOException { + StringBuilder sb = new StringBuilder(); + int data[] = node.getIntData(); + float floats[] = node.getFloatData(); + + // open three brackets + sb.append("((("); + // for each index, write the index and then its float + for (int i = 0; i < data.length; i++) { + sb.append("(" + data[i] + " " + floats[i] + ")"); + if (i + 1 != data.length) { + sb.append(" "); + } + } + // write the ending + sb.append(") 0))" + extension); + // dump the whole stuff + if (out != null) { + // write to output stream + + writeStringToOutput(sb.toString(), out); + } else { + // write to Standard out + // System.out.println(sb.toString()); + } + if (pw != null) { + // dump to printwriter + pw.println(sb.toString()); + } + } + + /** + * Writes the Cart to the given DataOut in Wagon Format + * + * @param out + * the outputStream + * @param extension + * the extension that is added to the last daughter + */ + private void toWagonFormat(IntArrayLeafNode node, DataOutputStream out, String extension, PrintWriter pw) throws IOException { + StringBuilder sb = new StringBuilder(); + int data[] = node.getIntData(); + + // open three brackets + sb.append("((("); + // for each index, write the index and then a pseudo float + for (int i = 0; i < data.length; i++) { + sb.append("(" + data[i] + " 0)"); + if (i + 1 != data.length) { + sb.append(" "); + } + } + // write the ending + sb.append(") 0))" + extension); + // dump the whole stuff + if (out != null) { + // write to output stream + + writeStringToOutput(sb.toString(), out); + } else { + // write to Standard out + // System.out.println(sb.toString()); + } + if (pw != null) { + // dump to printwriter + pw.println(sb.toString()); + } + } + + /** + * Write the given String to the given data output (Replacement for writeUTF) + * + * @param str + * the String + * @param out + * the data output + */ + private static void writeStringToOutput(String str, DataOutput out) throws IOException { + out.writeInt(str.length()); + out.writeChars(str); + } + +} diff --git a/marytts-unitselection/src/main/java/marytts/features/ByteValuedFeatureProcessor.java b/marytts-unitselection/src/main/java/marytts/features/ByteValuedFeatureProcessor.java new file mode 100644 index 0000000000..68c064e420 --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/features/ByteValuedFeatureProcessor.java @@ -0,0 +1,48 @@ +/** + * Portions Copyright 2006 DFKI GmbH. + * Portions Copyright 2001 Sun Microsystems, Inc. + * Portions Copyright 1999-2001 Language Technologies Institute, + * Carnegie Mellon University. + * All Rights Reserved. Use is subject to license terms. + * + * Permission is hereby granted, free of charge, to use and distribute + * this software and its documentation without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of this work, and to + * permit persons to whom this work is furnished to do so, subject to + * the following conditions: + * + * 1. The code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * 2. Any modifications must be clearly marked as such. + * 3. Original authors' names are not deleted. + * 4. The authors' names are not used to endorse or promote products + * derived from this software without specific prior written + * permission. + * + * DFKI GMBH AND THE CONTRIBUTORS TO THIS WORK DISCLAIM ALL WARRANTIES WITH + * REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL DFKI GMBH NOR THE + * CONTRIBUTORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL + * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR + * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS + * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF + * THIS SOFTWARE. + */ +package marytts.features; + + +/** + * Performs a specific type of processing on an item and returns an object. + */ +public interface ByteValuedFeatureProcessor extends MaryFeatureProcessor { + /** + * List the possible values of the feature processor, as clear-text values. Byte values as returned by process() can be + * translated into their string equivalent by using the byte value as an index in the String[] returned. + * + * @return an array containing the possible return values of this feature processor, in String representation. + */ + public String[] getValues(); + + public byte process(Target target); +} diff --git a/marytts-unitselection/src/main/java/marytts/features/ContinuousFeatureProcessor.java b/marytts-unitselection/src/main/java/marytts/features/ContinuousFeatureProcessor.java new file mode 100644 index 0000000000..afdb4667d6 --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/features/ContinuousFeatureProcessor.java @@ -0,0 +1,41 @@ +/** + * Portions Copyright 2006 DFKI GmbH. + * Portions Copyright 2001 Sun Microsystems, Inc. + * Portions Copyright 1999-2001 Language Technologies Institute, + * Carnegie Mellon University. + * All Rights Reserved. Use is subject to license terms. + * + * Permission is hereby granted, free of charge, to use and distribute + * this software and its documentation without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of this work, and to + * permit persons to whom this work is furnished to do so, subject to + * the following conditions: + * + * 1. The code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * 2. Any modifications must be clearly marked as such. + * 3. Original authors' names are not deleted. + * 4. The authors' names are not used to endorse or promote products + * derived from this software without specific prior written + * permission. + * + * DFKI GMBH AND THE CONTRIBUTORS TO THIS WORK DISCLAIM ALL WARRANTIES WITH + * REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL DFKI GMBH NOR THE + * CONTRIBUTORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL + * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR + * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS + * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF + * THIS SOFTWARE. + */ +package marytts.features; + + +/** + * Performs a specific type of processing on an item and returns an object. + */ +public interface ContinuousFeatureProcessor extends MaryFeatureProcessor { + + public float process(Target target); +} diff --git a/marytts-unitselection/src/main/java/marytts/features/DiphoneTarget.java b/marytts-unitselection/src/main/java/marytts/features/DiphoneTarget.java new file mode 100644 index 0000000000..a97566e37f --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/features/DiphoneTarget.java @@ -0,0 +1,71 @@ +/** + * Copyright 2006 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.features; + +import marytts.features.FeatureVector; +import marytts.modules.phonemiser.Allophone; + +import org.w3c.dom.Element; + +public class DiphoneTarget extends Target { + public final HalfPhoneTarget left; + public final HalfPhoneTarget right; + + public DiphoneTarget(HalfPhoneTarget left, HalfPhoneTarget right) { + super(null, null); + this.name = left.name.substring(0, left.name.lastIndexOf("_")) + "-" + + right.name.substring(0, right.name.lastIndexOf("_")); + assert left.isRightHalf(); // the left half of this diphone must be the right half of a phone + assert right.isLeftHalf(); + this.left = left; + this.right = right; + } + + @Override + public Element getMaryxmlElement() { + throw new IllegalStateException("This method should not be called for DiphoneTargets."); + } + + public FeatureVector getFeatureVector() { + throw new IllegalStateException("This method should not be called for DiphoneTargets."); + } + + public void setFeatureVector(FeatureVector featureVector) { + throw new IllegalStateException("This method should not be called for DiphoneTargets."); + } + + public float getTargetDurationInSeconds() { + throw new IllegalStateException("This method should not be called for DiphoneTargets."); + } + + /** + * Determine whether this target is a silence target + * + * @return true if the target represents silence, false otherwise + */ + public boolean isSilence() { + throw new IllegalStateException("This method should not be called for DiphoneTargets."); + } + + public Allophone getAllophone() { + throw new IllegalStateException("This method should not be called for DiphoneTargets."); + } + +} diff --git a/marytts-unitselection/src/main/java/marytts/features/FeatureDefinition.java b/marytts-unitselection/src/main/java/marytts/features/FeatureDefinition.java new file mode 100644 index 0000000000..00ddbaaf73 --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/features/FeatureDefinition.java @@ -0,0 +1,1730 @@ +/** + * Copyright 2006 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.features; + +import java.io.BufferedReader; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInput; +import java.io.DataInputStream; +import java.io.DataOutput; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.PrintWriter; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Set; + +import marytts.util.io.StreamUtils; +import marytts.util.string.ByteStringTranslator; +import marytts.util.string.IntStringTranslator; +import marytts.util.string.ShortStringTranslator; + +/** + * A feature definition object represents the "meaning" of feature vectors. It consists of a list of byte-valued, short-valued and + * continuous features by name and index position in the feature vector; the respective possible feature values (and corresponding + * byte and short codes); and, optionally, the weights and, for continuous features, weighting functions for each feature. + * + * @author Marc Schröder + * + */ +public class FeatureDefinition { + public static final String BYTEFEATURES = "ByteValuedFeatureProcessors"; + public static final String SHORTFEATURES = "ShortValuedFeatureProcessors"; + public static final String CONTINUOUSFEATURES = "ContinuousFeatureProcessors"; + public static final String FEATURESIMILARITY = "FeatureSimilarity"; + public static final char WEIGHT_SEPARATOR = '|'; + public static final String EDGEFEATURE = "edge"; + public static final String EDGEFEATURE_START = "start"; + public static final String EDGEFEATURE_END = "end"; + public static final String NULLVALUE = "0"; + + private int numByteFeatures; + private int numShortFeatures; + private int numContinuousFeatures; + private float[] featureWeights; + private IntStringTranslator featureNames; + // feature values: for byte and short features only + private ByteStringTranslator[] byteFeatureValues; + private ShortStringTranslator[] shortFeatureValues; + private String[] floatWeightFuncts; // for continuous features only + private float[][][] similarityMatrices = null; + + /** + * Create a feature definition object, reading textual data from the given BufferedReader. + * + * @param input + * a BufferedReader from which a textual feature definition can be read. + * @param readWeights + * a boolean indicating whether or not to read weights from input. If weights are read, they will be normalized so + * that they sum to one. + * @throws IOException + * if a reading problem occurs + * + */ + public FeatureDefinition(BufferedReader input, boolean readWeights) throws IOException { + // Section BYTEFEATURES + String line = input.readLine(); + if (line == null) + throw new IOException("Could not read from input"); + while (line.matches("^\\s*#.*") || line.matches("\\s*")) { + line = input.readLine(); + } + if (!line.trim().equals(BYTEFEATURES)) { + throw new IOException("Unexpected input: expected '" + BYTEFEATURES + "', read '" + line + "'"); + } + List byteFeatureLines = new ArrayList(); + while (true) { + line = input.readLine(); + if (line == null) + throw new IOException("Could not read from input"); + line = line.trim(); + if (line.equals(SHORTFEATURES)) + break; // Found end of section + byteFeatureLines.add(line); + } + // Section SHORTFEATURES + List shortFeatureLines = new ArrayList(); + while (true) { + line = input.readLine(); + if (line == null) + throw new IOException("Could not read from input"); + line = line.trim(); + if (line.equals(CONTINUOUSFEATURES)) + break; // Found end of section + shortFeatureLines.add(line); + } + // Section CONTINUOUSFEATURES + List continuousFeatureLines = new ArrayList(); + boolean readFeatureSimilarity = false; + while ((line = input.readLine()) != null) { // it's OK if we hit the end of the file now + line = line.trim(); + // if (line.equals(FEATURESIMILARITY) || line.equals("")) break; // Found end of section + if (line.equals(FEATURESIMILARITY)) { + // readFeatureSimilarityMatrices(input); + readFeatureSimilarity = true; + break; + } else if (line.equals("")) { // empty line: end of section + break; + } + continuousFeatureLines.add(line); + } + numByteFeatures = byteFeatureLines.size(); + numShortFeatures = shortFeatureLines.size(); + numContinuousFeatures = continuousFeatureLines.size(); + int total = numByteFeatures + numShortFeatures + numContinuousFeatures; + featureNames = new IntStringTranslator(total); + byteFeatureValues = new ByteStringTranslator[numByteFeatures]; + shortFeatureValues = new ShortStringTranslator[numShortFeatures]; + float sumOfWeights = 0; // for normalisation of weights + if (readWeights) { + featureWeights = new float[total]; + floatWeightFuncts = new String[numContinuousFeatures]; + } + + for (int i = 0; i < numByteFeatures; i++) { + line = byteFeatureLines.get(i); + String featureDef; + if (readWeights) { + int seppos = line.indexOf(WEIGHT_SEPARATOR); + if (seppos == -1) + throw new IOException("Weight separator '" + WEIGHT_SEPARATOR + "' not found in line '" + line + "'"); + String weightDef = line.substring(0, seppos).trim(); + featureDef = line.substring(seppos + 1).trim(); + // The weight definition is simply the float number: + featureWeights[i] = Float.parseFloat(weightDef); + sumOfWeights += featureWeights[i]; + if (featureWeights[i] < 0) + throw new IOException("Negative weight found in line '" + line + "'"); + } else { + featureDef = line; + } + // Now featureDef is a String in which the feature name and all feature values + // are separated by white space. + String[] nameAndValues = featureDef.split("\\s+", 2); + featureNames.set(i, nameAndValues[0]); // the feature name + byteFeatureValues[i] = new ByteStringTranslator(nameAndValues[1].split("\\s+")); // the feature values + } + + for (int i = 0; i < numShortFeatures; i++) { + line = shortFeatureLines.get(i); + String featureDef; + if (readWeights) { + int seppos = line.indexOf(WEIGHT_SEPARATOR); + if (seppos == -1) + throw new IOException("Weight separator '" + WEIGHT_SEPARATOR + "' not found in line '" + line + "'"); + String weightDef = line.substring(0, seppos).trim(); + featureDef = line.substring(seppos + 1).trim(); + // The weight definition is simply the float number: + featureWeights[numByteFeatures + i] = Float.parseFloat(weightDef); + sumOfWeights += featureWeights[numByteFeatures + i]; + if (featureWeights[numByteFeatures + i] < 0) + throw new IOException("Negative weight found in line '" + line + "'"); + } else { + featureDef = line; + } + // Now featureDef is a String in which the feature name and all feature values + // are separated by white space. + String[] nameAndValues = featureDef.split("\\s+", 2); + featureNames.set(numByteFeatures + i, nameAndValues[0]); // the feature name + shortFeatureValues[i] = new ShortStringTranslator(nameAndValues[1].split("\\s+")); // the feature values + } + + for (int i = 0; i < numContinuousFeatures; i++) { + line = continuousFeatureLines.get(i); + String featureDef; + if (readWeights) { + int seppos = line.indexOf(WEIGHT_SEPARATOR); + if (seppos == -1) + throw new IOException("Weight separator '" + WEIGHT_SEPARATOR + "' not found in line '" + line + "'"); + String weightDef = line.substring(0, seppos).trim(); + featureDef = line.substring(seppos + 1).trim(); + // The weight definition is the float number plus a definition of a weight function: + String[] weightAndFunction = weightDef.split("\\s+", 2); + featureWeights[numByteFeatures + numShortFeatures + i] = Float.parseFloat(weightAndFunction[0]); + sumOfWeights += featureWeights[numByteFeatures + numShortFeatures + i]; + if (featureWeights[numByteFeatures + numShortFeatures + i] < 0) + throw new IOException("Negative weight found in line '" + line + "'"); + try { + floatWeightFuncts[i] = weightAndFunction[1]; + } catch (ArrayIndexOutOfBoundsException e) { + // System.out.println( "weightDef string was: '" + weightDef + "'." ); + // System.out.println( "Splitting part 1: '" + weightAndFunction[0] + "'." ); + // System.out.println( "Splitting part 2: '" + weightAndFunction[1] + "'." ); + throw new RuntimeException("The string [" + weightDef + "] appears to be a badly formed" + + " weight plus weighting function definition."); + } + } else { + featureDef = line; + } + // Now featureDef is the feature name + // or the feature name followed by the word "float" + if (featureDef.endsWith("float")) { + String[] featureDefSplit = featureDef.split("\\s+", 2); + featureNames.set(numByteFeatures + numShortFeatures + i, featureDefSplit[0]); + } else { + featureNames.set(numByteFeatures + numShortFeatures + i, featureDef); + } + } + // Normalize weights to sum to one: + if (readWeights) { + for (int i = 0; i < total; i++) { + featureWeights[i] /= sumOfWeights; + } + } + + // read feature similarities here, if any + if (readFeatureSimilarity) { + readFeatureSimilarityMatrices(input); + } + } + + /** + * read similarity matrices from feature definition file + * + * @param input + * @throws IOException + */ + private void readFeatureSimilarityMatrices(BufferedReader input) throws IOException { + + String line = null; + + similarityMatrices = new float[this.getNumberOfByteFeatures()][][]; + for (int i = 0; i < this.getNumberOfByteFeatures(); i++) { + similarityMatrices[i] = null; + } + + while ((line = input.readLine()) != null) { + + if ("".equals(line)) { + return; + } + + String[] featureUniqueValues = line.trim().split("\\s+"); + String featureName = featureUniqueValues[0]; + + if (!isByteFeature(featureName)) { + throw new RuntimeException( + "Similarity matrix support is for bytefeatures only, but not for other feature types..."); + } + + int featureIndex = this.getFeatureIndex(featureName); + int noUniqValues = featureUniqueValues.length - 1; + similarityMatrices[featureIndex] = new float[noUniqValues][noUniqValues]; + + for (int i = 1; i <= noUniqValues; i++) { + + Arrays.fill(similarityMatrices[featureIndex][i - 1], 0); + String featureValue = featureUniqueValues[i]; + + String matLine = input.readLine(); + if (matLine == null) { + throw new RuntimeException("Feature definition file is having unexpected format..."); + } + + String[] lines = matLine.trim().split("\\s+"); + if (!featureValue.equals(lines[0])) { + throw new RuntimeException("Feature definition file is having unexpected format..."); + } + if (lines.length != i) { + throw new RuntimeException("Feature definition file is having unexpected format..."); + } + for (int j = 1; j < i; j++) { + float similarity = (new Float(lines[j])).floatValue(); + similarityMatrices[featureIndex][i - 1][j - 1] = similarity; + similarityMatrices[featureIndex][j - 1][i - 1] = similarity; + } + + } + } + + } + + /** + * Create a feature definition object, reading binary data from the given DataInput. + * + * @param input + * a DataInputStream or a RandomAccessFile from which a binary feature definition can be read. + * @throws IOException + * if a reading problem occurs + */ + public FeatureDefinition(DataInput input) throws IOException { + // Section BYTEFEATURES + numByteFeatures = input.readInt(); + byteFeatureValues = new ByteStringTranslator[numByteFeatures]; + // Initialise global arrays to byte feature length first; + // we have no means of knowing how many short or continuous + // features there will be, so we need to resize later. + // This will happen automatically for featureNames, but needs + // to be done by hand for featureWeights. + featureNames = new IntStringTranslator(numByteFeatures); + featureWeights = new float[numByteFeatures]; + // There is no need to normalise weights here, because + // they have already been normalized before the binary + // file was written. + for (int i = 0; i < numByteFeatures; i++) { + featureWeights[i] = input.readFloat(); + String featureName = input.readUTF(); + featureNames.set(i, featureName); + byte numberOfValuesEncoded = input.readByte(); // attention: this is an unsigned byte + int numberOfValues = numberOfValuesEncoded & 0xFF; + byteFeatureValues[i] = new ByteStringTranslator(numberOfValues); + for (int b = 0; b < numberOfValues; b++) { + String value = input.readUTF(); + byteFeatureValues[i].set((byte) b, value); + } + } + // Section SHORTFEATURES + numShortFeatures = input.readInt(); + if (numShortFeatures > 0) { + shortFeatureValues = new ShortStringTranslator[numShortFeatures]; + // resize weight array: + float[] newWeights = new float[numByteFeatures + numShortFeatures]; + System.arraycopy(featureWeights, 0, newWeights, 0, numByteFeatures); + featureWeights = newWeights; + + for (int i = 0; i < numShortFeatures; i++) { + featureWeights[numByteFeatures + i] = input.readFloat(); + String featureName = input.readUTF(); + featureNames.set(numByteFeatures + i, featureName); + short numberOfValues = input.readShort(); + shortFeatureValues[i] = new ShortStringTranslator(numberOfValues); + for (short s = 0; s < numberOfValues; s++) { + String value = input.readUTF(); + shortFeatureValues[i].set(s, value); + } + } + } + // Section CONTINUOUSFEATURES + numContinuousFeatures = input.readInt(); + floatWeightFuncts = new String[numContinuousFeatures]; + if (numContinuousFeatures > 0) { + // resize weight array: + float[] newWeights = new float[numByteFeatures + numShortFeatures + numContinuousFeatures]; + System.arraycopy(featureWeights, 0, newWeights, 0, numByteFeatures + numShortFeatures); + featureWeights = newWeights; + } + for (int i = 0; i < numContinuousFeatures; i++) { + featureWeights[numByteFeatures + numShortFeatures + i] = input.readFloat(); + floatWeightFuncts[i] = input.readUTF(); + String featureName = input.readUTF(); + featureNames.set(numByteFeatures + numShortFeatures + i, featureName); + } + } + + /** + * Create a feature definition object, reading binary data from the given byte buffer. + * + * @param input + * a byte buffer from which a binary feature definition can be read. + * @throws IOException + * if a reading problem occurs + */ + public FeatureDefinition(ByteBuffer bb) throws IOException { + // Section BYTEFEATURES + numByteFeatures = bb.getInt(); + byteFeatureValues = new ByteStringTranslator[numByteFeatures]; + // Initialise global arrays to byte feature length first; + // we have no means of knowing how many short or continuous + // features there will be, so we need to resize later. + // This will happen automatically for featureNames, but needs + // to be done by hand for featureWeights. + featureNames = new IntStringTranslator(numByteFeatures); + featureWeights = new float[numByteFeatures]; + // There is no need to normalise weights here, because + // they have already been normalized before the binary + // file was written. + for (int i = 0; i < numByteFeatures; i++) { + featureWeights[i] = bb.getFloat(); + String featureName = StreamUtils.readUTF(bb); + featureNames.set(i, featureName); + byte numberOfValuesEncoded = bb.get(); // attention: this is an unsigned byte + int numberOfValues = numberOfValuesEncoded & 0xFF; + byteFeatureValues[i] = new ByteStringTranslator(numberOfValues); + for (int b = 0; b < numberOfValues; b++) { + String value = StreamUtils.readUTF(bb); + byteFeatureValues[i].set((byte) b, value); + } + } + // Section SHORTFEATURES + numShortFeatures = bb.getInt(); + if (numShortFeatures > 0) { + shortFeatureValues = new ShortStringTranslator[numShortFeatures]; + // resize weight array: + float[] newWeights = new float[numByteFeatures + numShortFeatures]; + System.arraycopy(featureWeights, 0, newWeights, 0, numByteFeatures); + featureWeights = newWeights; + + for (int i = 0; i < numShortFeatures; i++) { + featureWeights[numByteFeatures + i] = bb.getFloat(); + String featureName = StreamUtils.readUTF(bb); + featureNames.set(numByteFeatures + i, featureName); + short numberOfValues = bb.getShort(); + shortFeatureValues[i] = new ShortStringTranslator(numberOfValues); + for (short s = 0; s < numberOfValues; s++) { + String value = StreamUtils.readUTF(bb); + shortFeatureValues[i].set(s, value); + } + } + } + // Section CONTINUOUSFEATURES + numContinuousFeatures = bb.getInt(); + floatWeightFuncts = new String[numContinuousFeatures]; + if (numContinuousFeatures > 0) { + // resize weight array: + float[] newWeights = new float[numByteFeatures + numShortFeatures + numContinuousFeatures]; + System.arraycopy(featureWeights, 0, newWeights, 0, numByteFeatures + numShortFeatures); + featureWeights = newWeights; + } + for (int i = 0; i < numContinuousFeatures; i++) { + featureWeights[numByteFeatures + numShortFeatures + i] = bb.getFloat(); + floatWeightFuncts[i] = StreamUtils.readUTF(bb); + String featureName = StreamUtils.readUTF(bb); + featureNames.set(numByteFeatures + numShortFeatures + i, featureName); + } + } + + /** + * Write this feature definition in binary format to the given output. + * + * @param out + * a DataOutputStream or RandomAccessFile to which the FeatureDefinition should be written. + * @throws IOException + * if a problem occurs while writing. + */ + public void writeBinaryTo(DataOutput out) throws IOException { + // TODO to avoid duplicate code, replace this with writeBinaryTo(out, List()) or some such + + // Section BYTEFEATURES + out.writeInt(numByteFeatures); + for (int i = 0; i < numByteFeatures; i++) { + if (featureWeights != null) { + out.writeFloat(featureWeights[i]); + } else { + out.writeFloat(0); + } + out.writeUTF(getFeatureName(i)); + + int numValues = getNumberOfValues(i); + byte numValuesEncoded = (byte) numValues; // an unsigned byte + out.writeByte(numValuesEncoded); + for (int b = 0; b < numValues; b++) { + String value = getFeatureValueAsString(i, b); + out.writeUTF(value); + } + } + // Section SHORTFEATURES + out.writeInt(numShortFeatures); + for (int i = numByteFeatures; i < numByteFeatures + numShortFeatures; i++) { + if (featureWeights != null) { + out.writeFloat(featureWeights[i]); + } else { + out.writeFloat(0); + } + out.writeUTF(getFeatureName(i)); + short numValues = (short) getNumberOfValues(i); + out.writeShort(numValues); + for (short b = 0; b < numValues; b++) { + String value = getFeatureValueAsString(i, b); + out.writeUTF(value); + } + } + // Section CONTINUOUSFEATURES + out.writeInt(numContinuousFeatures); + for (int i = numByteFeatures + numShortFeatures; i < numByteFeatures + numShortFeatures + numContinuousFeatures; i++) { + if (featureWeights != null) { + out.writeFloat(featureWeights[i]); + out.writeUTF(floatWeightFuncts[i - numByteFeatures - numShortFeatures]); + } else { + out.writeFloat(0); + out.writeUTF(""); + } + out.writeUTF(getFeatureName(i)); + } + } + + /** + * Write this feature definition in binary format to the given output, dropping featuresToDrop + * + * @param out + * a DataOutputStream or RandomAccessFile to which the FeatureDefinition should be written. + * @param featuresToDrop + * List of Integers containing the indices of features to drop from DataOutputStream + * @throws IOException + * if a problem occurs while writing. + */ + private void writeBinaryTo(DataOutput out, List featuresToDrop) throws IOException { + // how many features of each type are to be dropped + int droppedByteFeatures = 0; + int droppedShortFeatures = 0; + int droppedContinuousFeatures = 0; + for (int f : featuresToDrop) { + if (f < numByteFeatures) { + droppedByteFeatures++; + } else if (f < numByteFeatures + numShortFeatures) { + droppedShortFeatures++; + } else if (f < numByteFeatures + numShortFeatures + numContinuousFeatures) { + droppedContinuousFeatures++; + } + } + // Section BYTEFEATURES + out.writeInt(numByteFeatures - droppedByteFeatures); + for (int i = 0; i < numByteFeatures; i++) { + if (featuresToDrop.contains(i)) { + continue; + } + if (featureWeights != null) { + out.writeFloat(featureWeights[i]); + } else { + out.writeFloat(0); + } + out.writeUTF(getFeatureName(i)); + + int numValues = getNumberOfValues(i); + byte numValuesEncoded = (byte) numValues; // an unsigned byte + out.writeByte(numValuesEncoded); + for (int b = 0; b < numValues; b++) { + String value = getFeatureValueAsString(i, b); + out.writeUTF(value); + } + } + // Section SHORTFEATURES + out.writeInt(numShortFeatures - droppedShortFeatures); + for (int i = numByteFeatures; i < numByteFeatures + numShortFeatures; i++) { + if (featuresToDrop.contains(i)) { + continue; + } + if (featureWeights != null) { + out.writeFloat(featureWeights[i]); + } else { + out.writeFloat(0); + } + out.writeUTF(getFeatureName(i)); + short numValues = (short) getNumberOfValues(i); + out.writeShort(numValues); + for (short b = 0; b < numValues; b++) { + String value = getFeatureValueAsString(i, b); + out.writeUTF(value); + } + } + // Section CONTINUOUSFEATURES + out.writeInt(numContinuousFeatures - droppedContinuousFeatures); + for (int i = numByteFeatures + numShortFeatures; i < numByteFeatures + numShortFeatures + numContinuousFeatures; i++) { + if (featuresToDrop.contains(i)) { + continue; + } + if (featureWeights != null) { + out.writeFloat(featureWeights[i]); + out.writeUTF(floatWeightFuncts[i - numByteFeatures - numShortFeatures]); + } else { + out.writeFloat(0); + out.writeUTF(""); + } + out.writeUTF(getFeatureName(i)); + } + } + + /** + * Get the total number of features. + * + * @return the number of features + */ + public int getNumberOfFeatures() { + return numByteFeatures + numShortFeatures + numContinuousFeatures; + } + + /** + * Get the number of byte features. + * + * @return the number of features + */ + public int getNumberOfByteFeatures() { + return numByteFeatures; + } + + /** + * Get the number of short features. + * + * @return the number of features + */ + public int getNumberOfShortFeatures() { + return numShortFeatures; + } + + /** + * Get the number of continuous features. + * + * @return the number of features + */ + public int getNumberOfContinuousFeatures() { + return numContinuousFeatures; + } + + /** + * For the feature with the given index, return the weight. + * + * @param featureIndex + * @return a non-negative weight. + */ + public float getWeight(int featureIndex) { + return featureWeights[featureIndex]; + } + + public float[] getFeatureWeights() { + return featureWeights; + } + + /** + * Get the name of any weighting function associated with the given feature index. For byte-valued and short-valued features, + * this method will always return null; for continuous features, the method will return the name of a weighting function, or + * null. + * + * @param featureIndex + * @return the name of a weighting function, or null + */ + public String getWeightFunctionName(int featureIndex) { + return floatWeightFuncts[featureIndex - numByteFeatures - numShortFeatures]; + } + + // //////////////////// META-INFORMATION METHODS /////////////////////// + + /** + * Translate between a feature index and a feature name. + * + * @param index + * a feature index, as could be used to access a feature value in a FeatureVector. + * @return the name of the feature corresponding to the index + * @throws IndexOutOfBoundsException + * if index<0 or index>getNumberOfFeatures() + */ + public String getFeatureName(int index) { + return featureNames.get(index); + } + + /** + * Translate between an array of feature indexes and an array of feature names. + * + * @param index + * an array of feature indexes, as could be used to access a feature value in a FeatureVector. + * @return an array with the name of the features corresponding to the index + * @throws IndexOutOfBoundsException + * if any of the indexes is <0 or >getNumberOfFeatures() + */ + public String[] getFeatureNameArray(int[] index) { + String[] ret = new String[index.length]; + for (int i = 0; i < index.length; i++) { + ret[i] = getFeatureName(index[i]); + } + return (ret); + } + + /** + * Get names of all features + * + * @return an array of all feature name strings + */ + public String[] getFeatureNameArray() { + String[] names = new String[getNumberOfFeatures()]; + for (int i = 0; i < names.length; i++) { + names[i] = getFeatureName(i); + } + return (names); + } + + /** + * Get names of byte features + * + * @return an array of byte feature name strings + */ + public String[] getByteFeatureNameArray() { + String[] byteFeatureNames = new String[numByteFeatures]; + for (int i = 0; i < numByteFeatures; i++) { + assert isByteFeature(i); + byteFeatureNames[i] = getFeatureName(i); + } + return byteFeatureNames; + } + + /** + * Get names of short features + * + * @return an array of short feature name strings + */ + public String[] getShortFeatureNameArray() { + String[] shortFeatureNames = new String[numShortFeatures]; + for (int i = 0; i < numShortFeatures; i++) { + int shortFeatureIndex = numByteFeatures + i; + assert isShortFeature(shortFeatureIndex); + shortFeatureNames[i] = getFeatureName(shortFeatureIndex); + } + return shortFeatureNames; + } + + /** + * Get names of continuous features + * + * @return an array of continuous feature name strings + */ + public String[] getContinuousFeatureNameArray() { + String[] continuousFeatureNames = new String[numContinuousFeatures]; + for (int i = 0; i < numContinuousFeatures; i++) { + int continuousFeatureIndex = numByteFeatures + numShortFeatures + i; + assert isContinuousFeature(continuousFeatureIndex); + continuousFeatureNames[i] = getFeatureName(continuousFeatureIndex); + } + return continuousFeatureNames; + } + + /** + * List all feature names, separated by white space, in their order of definition. + * + * @return + */ + public String getFeatureNames() { + StringBuilder buf = new StringBuilder(); + for (int i = 0, n = getNumberOfFeatures(); i < n; i++) { + if (buf.length() > 0) + buf.append(" "); + buf.append(featureNames.get(i)); + } + return buf.toString(); + } + + /** + * Indicate whether the feature definition contains the feature with the given name + * + * @param name + * the feature name in question, e.g. "next_next_phone" + * @return + */ + public boolean hasFeature(String name) { + return featureNames.contains(name); + } + + /** + * Query a feature as identified by the given featureName as to whether the given featureValue is a known value of that + * feature. In other words, this will return true exactly if the given feature is a byte feature and + * getFeatureValueAsByte(featureName, featureValue) will not throw an exception or if the given feature is a short feature and + * getFeatureValueAsShort(featureName, featureValue) will not throw an exception. + * + * @param featureName + * @param featureValue + * @return + */ + public boolean hasFeatureValue(String featureName, String featureValue) { + return hasFeatureValue(getFeatureIndex(featureName), featureValue); + } + + /** + * Query a feature as identified by the given featureIndex as to whether the given featureValue is a known value of that + * feature. In other words, this will return true exactly if the given feature is a byte feature and + * getFeatureValueAsByte(featureIndex, featureValue) will not throw an exception or if the given feature is a short feature + * and getFeatureValueAsShort(featureIndex, featureValue) will not throw an exception. + * + * @param featureIndex + * @param featureValue + * @return + */ + public boolean hasFeatureValue(int featureIndex, String featureValue) { + if (featureIndex < 0) { + return false; + } + if (featureIndex < numByteFeatures) { + return byteFeatureValues[featureIndex].contains(featureValue); + } + if (featureIndex < numByteFeatures + numShortFeatures) { + return shortFeatureValues[featureIndex - numByteFeatures].contains(featureValue); + } + return false; + } + + /** + * Determine whether the feature with the given name is a byte feature. + * + * @param featureName + * @return true if the feature is a byte feature, false if the feature is not known or is not a byte feature + */ + public boolean isByteFeature(String featureName) { + try { + int index = getFeatureIndex(featureName); + return isByteFeature(index); + } catch (Exception e) { + return false; + } + } + + /** + * Determine whether the feature with the given index number is a byte feature. + * + * @param featureIndex + * @return true if the feature is a byte feature, false if the feature is not a byte feature or is invalid + */ + public boolean isByteFeature(int index) { + return 0 <= index && index < numByteFeatures; + } + + /** + * Determine whether the feature with the given name is a short feature. + * + * @param featureName + * @return true if the feature is a short feature, false if the feature is not known or is not a short feature + */ + public boolean isShortFeature(String featureName) { + try { + int index = getFeatureIndex(featureName); + return isShortFeature(index); + } catch (Exception e) { + return false; + } + } + + /** + * Determine whether the feature with the given index number is a short feature. + * + * @param featureIndex + * @return true if the feature is a short feature, false if the feature is not a short feature or is invalid + */ + public boolean isShortFeature(int index) { + index -= numByteFeatures; + return 0 <= index && index < numShortFeatures; + } + + /** + * Determine whether the feature with the given name is a continuous feature. + * + * @param featureName + * @return true if the feature is a continuous feature, false if the feature is not known or is not a continuous feature + */ + public boolean isContinuousFeature(String featureName) { + try { + int index = getFeatureIndex(featureName); + return isContinuousFeature(index); + } catch (Exception e) { + return false; + } + } + + /** + * Determine whether the feature with the given index number is a continuous feature. + * + * @param featureIndex + * @return true if the feature is a continuous feature, false if the feature is not a continuous feature or is invalid + */ + public boolean isContinuousFeature(int index) { + index -= numByteFeatures; + index -= numShortFeatures; + return 0 <= index && index < numContinuousFeatures; + } + + /** + * true, if given feature index contains similarity matrix + * + * @param featureIndex + * @return + */ + public boolean hasSimilarityMatrix(int featureIndex) { + + if (featureIndex >= this.getNumberOfByteFeatures()) { + return false; + } + if (this.similarityMatrices != null && this.similarityMatrices[featureIndex] != null) { + return true; + } + return false; + } + + /** + * true, if given feature name contains similarity matrix + * + * @param featureName + * @return + */ + public boolean hasSimilarityMatrix(String featureName) { + return hasSimilarityMatrix(this.getFeatureIndex(featureName)); + } + + /** + * To get a similarity between two feature values + * + * @param featureIndex + * @param i + * @param j + * @return + */ + public float getSimilarity(int featureIndex, byte i, byte j) { + if (!hasSimilarityMatrix(featureIndex)) { + throw new RuntimeException("the given feature index "); + } + return this.similarityMatrices[featureIndex][i][j]; + } + + /** + * Translate between a feature name and a feature index. + * + * @param featureName + * a valid feature name + * @return a feature index, as could be used to access a feature value in a FeatureVector. + * @throws IllegalArgumentException + * if the feature name is unknown. + */ + public int getFeatureIndex(String featureName) { + return featureNames.get(featureName); + } + + /** + * Translate between an array of feature names and an array of feature indexes. + * + * @param featureName + * an array of valid feature names + * @return an array of feature indexes, as could be used to access a feature value in a FeatureVector. + * @throws IllegalArgumentException + * if one of the feature names is unknown. + */ + public int[] getFeatureIndexArray(String[] featureName) { + int[] ret = new int[featureName.length]; + for (int i = 0; i < featureName.length; i++) { + ret[i] = getFeatureIndex(featureName[i]); + } + return (ret); + } + + /** + * Get the number of possible values for the feature with the given index number. This method must only be called for + * byte-valued or short-valued features. + * + * @param featureIndex + * the index number of the feature. + * @return for byte-valued and short-valued features, return the number of values. + * @throws IndexOutOfBoundsException + * if featureIndex < 0 or featureIndex >= getNumberOfByteFeatures() + getNumberOfShortFeatures(). + */ + public int getNumberOfValues(int featureIndex) { + if (featureIndex < numByteFeatures) + return byteFeatureValues[featureIndex].getNumberOfValues(); + featureIndex -= numByteFeatures; + if (featureIndex < numShortFeatures) + return shortFeatureValues[featureIndex].getNumberOfValues(); + throw new IndexOutOfBoundsException("Feature no. " + featureIndex + " is not a byte-valued or short-valued feature"); + } + + /** + * Get the list of possible String values for the feature with the given index number. This method must only be called for + * byte-valued or short-valued features. The position in the String array corresponds to the byte or short value of the + * feature obtained from a FeatureVector. + * + * @param featureIndex + * the index number of the feature. + * @return for byte-valued and short-valued features, return the array of String values. + * @throws IndexOutOfBoundsException + * if featureIndex < 0 or featureIndex >= getNumberOfByteFeatures() + getNumberOfShortFeatures(). + */ + public String[] getPossibleValues(int featureIndex) { + if (featureIndex < numByteFeatures) + return byteFeatureValues[featureIndex].getStringValues(); + featureIndex -= numByteFeatures; + if (featureIndex < numShortFeatures) + return shortFeatureValues[featureIndex].getStringValues(); + throw new IndexOutOfBoundsException("Feature no. " + featureIndex + " is not a byte-valued or short-valued feature"); + } + + /** + * For the feature with the given index number, translate its byte or short value to its String value. This method must only + * be called for byte-valued or short-valued features. + * + * @param featureIndex + * the index number of the feature. + * @param value + * the feature value. This must be in the range of acceptable values for the given feature. + * @return for byte-valued and short-valued features, return the String representation of the feature value. + * @throws IndexOutOfBoundsException + * if featureIndex < 0 or featureIndex >= getNumberOfByteFeatures() + getNumberOfShortFeatures() + * @throws IndexOutOfBoundsException + * if value is not a legal value for this feature + * + * + */ + public String getFeatureValueAsString(int featureIndex, int value) { + if (featureIndex < numByteFeatures) + return byteFeatureValues[featureIndex].get((byte) value); + featureIndex -= numByteFeatures; + if (featureIndex < numShortFeatures) + return shortFeatureValues[featureIndex].get((short) value); + throw new IndexOutOfBoundsException("Feature no. " + featureIndex + " is not a byte-valued or short-valued feature"); + } + + /** + * Simple access to string-based features. + * + * @param featureName + * @param fv + * @return + */ + public String getFeatureValueAsString(String featureName, FeatureVector fv) { + int i = getFeatureIndex(featureName); + return getFeatureValueAsString(i, fv.getFeatureAsInt(i)); + } + + /** + * For the feature with the given name, translate its String value to its byte value. This method must only be called for + * byte-valued features. + * + * @param featureName + * the name of the feature. + * @param value + * the feature value. This must be among the acceptable values for the given feature. + * @return for byte-valued features, return the byte representation of the feature value. + * @throws IllegalArgumentException + * if featureName is not a valid feature name, or if featureName is not a byte-valued feature. + * @throws IllegalArgumentException + * if value is not a legal value for this feature + */ + public byte getFeatureValueAsByte(String featureName, String value) { + int featureIndex = getFeatureIndex(featureName); + return getFeatureValueAsByte(featureIndex, value); + } + + /** + * For the feature with the given index number, translate its String value to its byte value. This method must only be called + * for byte-valued features. + * + * @param featureName + * the name of the feature. + * @param value + * the feature value. This must be among the acceptable values for the given feature. + * @return for byte-valued features, return the byte representation of the feature value. + * @throws IllegalArgumentException + * if featureName is not a valid feature name, or if featureName is not a byte-valued feature. + * @throws IllegalArgumentException + * if value is not a legal value for this feature + */ + public byte getFeatureValueAsByte(int featureIndex, String value) { + if (featureIndex >= numByteFeatures) + throw new IndexOutOfBoundsException("Feature no. " + featureIndex + " is not a byte-valued feature"); + try { + return byteFeatureValues[featureIndex].get(value); + } catch (IllegalArgumentException iae) { + StringBuilder message = new StringBuilder("Illegal value '" + value + "' for feature " + getFeatureName(featureIndex) + + "; Legal values are:\n"); + for (String v : getPossibleValues(featureIndex)) { + message.append(" " + v); + } + throw new IllegalArgumentException(message.toString()); + } + + } + + /** + * For the feature with the given name, translate its String value to its short value. This method must only be called for + * short-valued features. + * + * @param featureName + * the name of the feature. + * @param value + * the feature value. This must be among the acceptable values for the given feature. + * @return for short-valued features, return the short representation of the feature value. + * @throws IllegalArgumentException + * if featureName is not a valid feature name, or if featureName is not a short-valued feature. + * @throws IllegalArgumentException + * if value is not a legal value for this feature + */ + public short getFeatureValueAsShort(String featureName, String value) { + int featureIndex = getFeatureIndex(featureName); + featureIndex -= numByteFeatures; + if (featureIndex < numShortFeatures) + return shortFeatureValues[featureIndex].get(value); + throw new IndexOutOfBoundsException("Feature '" + featureName + "' is not a short-valued feature"); + } + + /** + * For the feature with the given name, translate its String value to its short value. This method must only be called for + * short-valued features. + * + * @param featureName + * the name of the feature. + * @param value + * the feature value. This must be among the acceptable values for the given feature. + * @return for short-valued features, return the short representation of the feature value. + * @throws IllegalArgumentException + * if featureName is not a valid feature name, or if featureName is not a short-valued feature. + * @throws IllegalArgumentException + * if value is not a legal value for this feature + */ + public short getFeatureValueAsShort(int featureIndex, String value) { + featureIndex -= numByteFeatures; + if (featureIndex < numShortFeatures) + return shortFeatureValues[featureIndex].get(value); + throw new IndexOutOfBoundsException("Feature no. " + featureIndex + " is not a short-valued feature"); + } + + /** + * Determine whether two feature definitions are equal, with respect to number, names, and possible values of the three kinds + * of features (byte-valued, short-valued, continuous). This method does not compare any weights. + * + * @param other + * the feature definition to compare to + * @return true if all features and values are identical, false otherwise + */ + public boolean featureEquals(FeatureDefinition other) { + if (numByteFeatures != other.numByteFeatures || numShortFeatures != other.numShortFeatures + || numContinuousFeatures != other.numContinuousFeatures) + return false; + // Compare the feature names and values for byte and short features: + for (int i = 0; i < numByteFeatures + numShortFeatures + numContinuousFeatures; i++) { + if (!getFeatureName(i).equals(other.getFeatureName(i))) + return false; + } + // Compare the values for byte and short features: + for (int i = 0; i < numByteFeatures + numShortFeatures; i++) { + if (getNumberOfValues(i) != other.getNumberOfValues(i)) + return false; + for (int v = 0, n = getNumberOfValues(i); v < n; v++) { + if (!getFeatureValueAsString(i, v).equals(other.getFeatureValueAsString(i, v))) + return false; + } + } + return true; + } + + /** + * An extension of the previous method. + */ + public String featureEqualsAnalyse(FeatureDefinition other) { + if (numByteFeatures != other.numByteFeatures) { + return ("The number of BYTE features differs: " + numByteFeatures + " versus " + other.numByteFeatures); + } + if (numShortFeatures != other.numShortFeatures) { + return ("The number of SHORT features differs: " + numShortFeatures + " versus " + other.numShortFeatures); + } + if (numContinuousFeatures != other.numContinuousFeatures) { + return ("The number of CONTINUOUS features differs: " + numContinuousFeatures + " versus " + other.numContinuousFeatures); + } + // Compare the feature names and values for byte and short features: + for (int i = 0; i < numByteFeatures + numShortFeatures + numContinuousFeatures; i++) { + if (!getFeatureName(i).equals(other.getFeatureName(i))) { + return ("The feature name differs at position [" + i + "]: " + getFeatureName(i) + " versus " + other + .getFeatureName(i)); + } + } + // Compare the values for byte and short features: + for (int i = 0; i < numByteFeatures + numShortFeatures; i++) { + if (getNumberOfValues(i) != other.getNumberOfValues(i)) { + return ("The number of values differs at position [" + i + "]: " + getNumberOfValues(i) + " versus " + other + .getNumberOfValues(i)); + } + for (int v = 0, n = getNumberOfValues(i); v < n; v++) { + if (!getFeatureValueAsString(i, v).equals(other.getFeatureValueAsString(i, v))) { + return ("The feature value differs at position [" + i + "] for feature value [" + v + "]: " + + getFeatureValueAsString(i, v) + " versus " + other.getFeatureValueAsString(i, v)); + } + } + } + return ""; + } + + /** + * Determine whether two feature definitions are equal, regarding both the actual feature definitions and the weights. The + * comparison of weights will succeed if both have no weights or if both have exactly the same weights + * + * @param other + * the feature definition to compare to + * @return true if all features, values and weights are identical, false otherwise + * @see #featureEquals(FeatureDefinition) + */ + @Override + public boolean equals(Object obj) { + if (!(obj instanceof FeatureDefinition)) + return false; + FeatureDefinition other = (FeatureDefinition) obj; + if (featureWeights == null) { + if (other.featureWeights != null) + return false; + // Both are null + } else { // featureWeights != null + if (other.featureWeights == null) + return false; + // Both != null + if (featureWeights.length != other.featureWeights.length) + return false; + for (int i = 0; i < featureWeights.length; i++) { + if (featureWeights[i] != other.featureWeights[i]) + return false; + } + assert floatWeightFuncts != null; + assert other.floatWeightFuncts != null; + if (floatWeightFuncts.length != other.floatWeightFuncts.length) + return false; + for (int i = 0; i < floatWeightFuncts.length; i++) { + if (floatWeightFuncts[i] == null) { + if (other.floatWeightFuncts[i] != null) + return false; + // Both are null + } else { // != null + if (other.floatWeightFuncts[i] == null) + return false; + // Both != null + if (!floatWeightFuncts[i].equals(other.floatWeightFuncts[i])) + return false; + } + } + } + // OK, weights are equal + return featureEquals(other); + } + + /** + * Determine whether this FeatureDefinition is a superset of, or equal to, another FeatureDefinition. + *

+ * Specifically, + *

    + *
  1. every byte-valued feature in other must be in this, likewise for short-valued and continuous-valued + * features;
    2. + *
  2. for byte-valued and short-valued features, the possible feature values must be the same in this and + * other.
    3. + *
+ * + * @param other + * FeatureDefinition + * @return true if + *
    + *
  1. all features in other are also in this, and every feature in other is of the same type in + * this; and
  2. every feature in other has the same possible values as the feature in this + *
    3. + *
+ * false otherwise + * @author steiner + */ + public boolean contains(FeatureDefinition other) { + List thisByteFeatures = Arrays.asList(this.getByteFeatureNameArray()); + List otherByteFeatures = Arrays.asList(other.getByteFeatureNameArray()); + if (!thisByteFeatures.containsAll(otherByteFeatures)) { + return false; + } + for (String commonByteFeature : otherByteFeatures) { + String[] thisByteFeaturePossibleValues = this.getPossibleValues(this.getFeatureIndex(commonByteFeature)); + String[] otherByteFeaturePossibleValues = other.getPossibleValues(other.getFeatureIndex(commonByteFeature)); + if (!Arrays.equals(thisByteFeaturePossibleValues, otherByteFeaturePossibleValues)) { + return false; + } + } + List thisShortFeatures = Arrays.asList(this.getShortFeatureNameArray()); + List otherShortFeatures = Arrays.asList(other.getShortFeatureNameArray()); + if (!thisShortFeatures.containsAll(otherShortFeatures)) { + return false; + } + for (String commonShortFeature : otherShortFeatures) { + String[] thisShortFeaturePossibleValues = this.getPossibleValues(this.getFeatureIndex(commonShortFeature)); + String[] otherShortFeaturePossibleValues = other.getPossibleValues(other.getFeatureIndex(commonShortFeature)); + if (!Arrays.equals(thisShortFeaturePossibleValues, otherShortFeaturePossibleValues)) { + return false; + } + } + List thisContinuousFeatures = Arrays.asList(this.getContinuousFeatureNameArray()); + List otherContinuousFeatures = Arrays.asList(other.getContinuousFeatureNameArray()); + if (!thisContinuousFeatures.containsAll(otherContinuousFeatures)) { + return false; + } + return true; + } + + /** + * Create a new FeatureDefinition that contains a subset of the features in this. + * + * @param featureNamesToDrop + * array of Strings containing the names of the features to drop from the new FeatureDefinition + * @return new FeatureDefinition + * @author steiner + */ + public FeatureDefinition subset(String[] featureNamesToDrop) { + // construct a list of indices for the features to be dropped: + List featureIndicesToDrop = new ArrayList(); + for (String featureName : featureNamesToDrop) { + int featureIndex; + try { + featureIndex = getFeatureIndex(featureName); + featureIndicesToDrop.add(featureIndex); + } catch (IllegalArgumentException e) { + System.err.println("WARNING: feature " + featureName + " not found in FeatureDefinition; ignoring."); + } + } + + // create a new FeatureDefinition by way of a byte array: + FeatureDefinition subDefinition = null; + try { + ByteArrayOutputStream toMemory = new ByteArrayOutputStream(); + DataOutput output = new DataOutputStream(toMemory); + writeBinaryTo(output, featureIndicesToDrop); + + byte[] memory = toMemory.toByteArray(); + + ByteArrayInputStream fromMemory = new ByteArrayInputStream(memory); + DataInput input = new DataInputStream(fromMemory); + + subDefinition = new FeatureDefinition(input); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + + // make sure that subDefinition really is a subset of this + assert this.contains(subDefinition); + + return subDefinition; + } + + /** + * Create a feature vector consistent with this feature definition by reading the data from a String representation. In that + * String, the String values for each feature must be separated by white space. For example, this format is created by + * toFeatureString(FeatureVector). + * + * @param unitIndex + * an index number to assign to the feature vector + * @param featureString + * the string representation of a feature vector. + * @return the feature vector created from the String. + * @throws IllegalArgumentException + * if the feature values listed are not consistent with the feature definition. + * @see #toFeatureString(FeatureVector) + */ + public FeatureVector toFeatureVector(int unitIndex, String featureString) { + String[] featureValues = featureString.split("\\s+"); + if (featureValues.length != numByteFeatures + numShortFeatures + numContinuousFeatures) + throw new IllegalArgumentException("Expected " + (numByteFeatures + numShortFeatures + numContinuousFeatures) + + " features, got " + featureValues.length); + byte[] bytes = new byte[numByteFeatures]; + short[] shorts = new short[numShortFeatures]; + float[] floats = new float[numContinuousFeatures]; + for (int i = 0; i < numByteFeatures; i++) { + bytes[i] = Byte.parseByte(featureValues[i]); + } + for (int i = 0; i < numShortFeatures; i++) { + shorts[i] = Short.parseShort(featureValues[numByteFeatures + i]); + } + for (int i = 0; i < numContinuousFeatures; i++) { + floats[i] = Float.parseFloat(featureValues[numByteFeatures + numShortFeatures + i]); + } + return new FeatureVector(bytes, shorts, floats, unitIndex); + } + + public FeatureVector toFeatureVector(int unitIndex, byte[] bytes, short[] shorts, float[] floats) { + if (!((numByteFeatures == 0 && bytes == null || numByteFeatures == bytes.length) + && (numShortFeatures == 0 && shorts == null || numShortFeatures == shorts.length) && (numContinuousFeatures == 0 + && floats == null || numContinuousFeatures == floats.length))) { + throw new IllegalArgumentException("Expected " + numByteFeatures + " bytes (got " + + (bytes == null ? "0" : bytes.length) + "), " + numShortFeatures + " shorts (got " + + (shorts == null ? "0" : shorts.length) + "), " + numContinuousFeatures + " floats (got " + + (floats == null ? "0" : floats.length) + ")"); + } + return new FeatureVector(bytes, shorts, floats, unitIndex); + } + + /** + * Create a feature vector consistent with this feature definition by reading the data from the given input. + * + * @param input + * a DataInputStream or RandomAccessFile to read the feature values from. + * @return a FeatureVector. + */ + public FeatureVector readFeatureVector(int currentUnitIndex, DataInput input) throws IOException { + byte[] bytes = new byte[numByteFeatures]; + input.readFully(bytes); + short[] shorts = new short[numShortFeatures]; + for (int i = 0; i < shorts.length; i++) { + shorts[i] = input.readShort(); + } + float[] floats = new float[numContinuousFeatures]; + for (int i = 0; i < floats.length; i++) { + floats[i] = input.readFloat(); + } + return new FeatureVector(bytes, shorts, floats, currentUnitIndex); + } + + /** + * Create a feature vector consistent with this feature definition by reading the data from the byte buffer. + * + * @param bb + * a byte buffer to read the feature values from. + * @return a FeatureVector. + */ + public FeatureVector readFeatureVector(int currentUnitIndex, ByteBuffer bb) throws IOException { + byte[] bytes = new byte[numByteFeatures]; + bb.get(bytes); + short[] shorts = new short[numShortFeatures]; + for (int i = 0; i < shorts.length; i++) { + shorts[i] = bb.getShort(); + } + float[] floats = new float[numContinuousFeatures]; + for (int i = 0; i < floats.length; i++) { + floats[i] = bb.getFloat(); + } + return new FeatureVector(bytes, shorts, floats, currentUnitIndex); + } + + /** + * Create a feature vector that marks a start or end of a unit. All feature values are set to the neutral value "0", except + * for the EDGEFEATURE, which is set to start if start == true, to end otherwise. + * + * @param unitIndex + * index of the unit + * @param start + * true creates a start vector, false creates an end vector. + * @return a feature vector representing an edge. + */ + public FeatureVector createEdgeFeatureVector(int unitIndex, boolean start) { + int edgeFeature = getFeatureIndex(EDGEFEATURE); + assert edgeFeature < numByteFeatures; // we can assume this is byte-valued + byte edge; + if (start) + edge = getFeatureValueAsByte(edgeFeature, EDGEFEATURE_START); + else + edge = getFeatureValueAsByte(edgeFeature, EDGEFEATURE_END); + byte[] bytes = new byte[numByteFeatures]; + short[] shorts = new short[numShortFeatures]; + float[] floats = new float[numContinuousFeatures]; + for (int i = 0; i < numByteFeatures; i++) { + bytes[i] = getFeatureValueAsByte(i, NULLVALUE); + } + for (int i = 0; i < numShortFeatures; i++) { + shorts[i] = getFeatureValueAsShort(numByteFeatures + i, NULLVALUE); + } + for (int i = 0; i < numContinuousFeatures; i++) { + floats[i] = 0; + } + bytes[edgeFeature] = edge; + return new FeatureVector(bytes, shorts, floats, unitIndex); + } + + /** + * Convert a feature vector into a String representation. + * + * @param fv + * a feature vector which must be consistent with this feature definition. + * @return a String containing the String values of all features, separated by white space. + * @throws IllegalArgumentException + * if the feature vector is not consistent with this feature definition + * @throws IndexOutOfBoundsException + * if any value of the feature vector is not consistent with this feature definition + */ + public String toFeatureString(FeatureVector fv) { + if (numByteFeatures != fv.getNumberOfByteFeatures() || numShortFeatures != fv.getNumberOfShortFeatures() + || numContinuousFeatures != fv.getNumberOfContinuousFeatures()) + throw new IllegalArgumentException("Feature vector '" + fv + "' is inconsistent with feature definition"); + StringBuilder buf = new StringBuilder(); + for (int i = 0; i < numByteFeatures; i++) { + if (buf.length() > 0) + buf.append(" "); + buf.append(getFeatureValueAsString(i, fv.getByteFeature(i))); + } + for (int i = numByteFeatures; i < numByteFeatures + numShortFeatures; i++) { + if (buf.length() > 0) + buf.append(" "); + buf.append(getFeatureValueAsString(i, fv.getShortFeature(i))); + } + for (int i = numByteFeatures + numShortFeatures; i < numByteFeatures + numShortFeatures + numContinuousFeatures; i++) { + if (buf.length() > 0) + buf.append(" "); + buf.append(fv.getContinuousFeature(i)); + } + return buf.toString(); + } + + /** + * Export this feature definition in the text format which can also be read by this class. + * + * @param out + * the destination of the data + * @param writeWeights + * whether to write weights before every line + */ + public void writeTo(PrintWriter out, boolean writeWeights) { + out.println("ByteValuedFeatureProcessors"); + for (int i = 0; i < numByteFeatures; i++) { + if (writeWeights) { + out.print(featureWeights[i] + " | "); + } + out.print(getFeatureName(i)); + for (int v = 0, vmax = getNumberOfValues(i); v < vmax; v++) { + out.print(" "); + String val = getFeatureValueAsString(i, v); + out.print(val); + } + out.println(); + } + out.println("ShortValuedFeatureProcessors"); + for (int i = 0; i < numShortFeatures; i++) { + if (writeWeights) { + out.print(featureWeights[numByteFeatures + i] + " | "); + } + out.print(getFeatureName(numByteFeatures + i)); + for (int v = 0, vmax = getNumberOfValues(numByteFeatures + i); v < vmax; v++) { + out.print(" "); + String val = getFeatureValueAsString(numByteFeatures + i, v); + out.print(val); + } + out.println(); + } + out.println("ContinuousFeatureProcessors"); + for (int i = 0; i < numContinuousFeatures; i++) { + if (writeWeights) { + out.print(featureWeights[numByteFeatures + numShortFeatures + i]); + out.print(" "); + out.print(floatWeightFuncts[i]); + out.print(" | "); + } + + out.print(getFeatureName(numByteFeatures + numShortFeatures + i)); + out.println(); + } + + } + + /** + * Export this feature definition in the "all.desc" format which can be read by wagon. + * + * @param out + * the destination of the data + */ + public void generateAllDotDescForWagon(PrintWriter out) { + generateAllDotDescForWagon(out, null); + } + + /** + * Export this feature definition in the "all.desc" format which can be read by wagon. + * + * @param out + * the destination of the data + * @param featuresToIgnore + * a set of Strings containing the names of features that wagon should ignore. Can be null. + */ + public void generateAllDotDescForWagon(PrintWriter out, Set featuresToIgnore) { + out.println("("); + out.println("(occurid cluster)"); + for (int i = 0, n = getNumberOfFeatures(); i < n; i++) { + out.print("( "); + String featureName = getFeatureName(i); + out.print(featureName); + if (featuresToIgnore != null && featuresToIgnore.contains(featureName)) { + out.print(" ignore"); + } + if (i < numByteFeatures + numShortFeatures) { // list values + for (int v = 0, vmax = getNumberOfValues(i); v < vmax; v++) { + out.print(" "); + // Print values surrounded by double quotes, and make sure any + // double quotes in the value are preceded by a backslash -- + // otherwise, we get problems e.g. for sentence_punc + String val = getFeatureValueAsString(i, v); + if (val.indexOf('"') != -1) { + StringBuilder buf = new StringBuilder(); + for (int c = 0; c < val.length(); c++) { + char ch = val.charAt(c); + if (ch == '"') + buf.append("\\\""); + else + buf.append(ch); + } + val = buf.toString(); + } + out.print("\"" + val + "\""); + } + + out.println(" )"); + } else { // float feature + out.println(" float )"); + } + + } + out.println(")"); + } + + /** + * Print this feature definition plus weights to a .txt file + * + * @param out + * the destination of the data + */ + public void generateFeatureWeightsFile(PrintWriter out) { + out.println("# This file lists the features and their weights to be used for\n" + "# creating the MARY features file.\n" + + "# The same file can also be used to override weights in a run-time system.\n" + + "# Three sections are distinguished: Byte-valued, Short-valued, and\n" + "# Continuous features.\n" + "#\n" + + "# Lines starting with '#' are ignored; they can be used for comments\n" + + "# anywhere in the file. Empty lines are also ignored.\n" + "# Entries must have the following form:\n" + + "# \n" + "# | \n" + "# \n" + + "# For byte and short features, is simply the \n" + + "# (float) number representing the weight.\n" + "# For continuous features, is the\n" + + "# (float) number representing the weight, followed by an optional\n" + + "# weighting function including arguments.\n" + "#\n" + + "# The is the feature name, which in the case of\n" + + "# byte and short features is followed by the full list of feature values.\n" + "#\n" + + "# Note that the feature definitions must be identical between this file\n" + + "# and all unit feature files for individual database utterances.\n" + + "# THIS FILE WAS GENERATED AUTOMATICALLY"); + out.println(); + out.println("ByteValuedFeatureProcessors"); + List getValuesOf10 = new ArrayList(); + getValuesOf10.add("phone"); + getValuesOf10.add("ph_vc"); + getValuesOf10.add("prev_phone"); + getValuesOf10.add("next_phone"); + getValuesOf10.add("stressed"); + getValuesOf10.add("syl_break"); + getValuesOf10.add("prev_syl_break"); + getValuesOf10.add("next_is_pause"); + getValuesOf10.add("prev_is_pause"); + List getValuesOf5 = new ArrayList(); + getValuesOf5.add("cplace"); + getValuesOf5.add("ctype"); + getValuesOf5.add("cvox"); + getValuesOf5.add("vfront"); + getValuesOf5.add("vheight"); + getValuesOf5.add("vlng"); + getValuesOf5.add("vrnd"); + getValuesOf5.add("vc"); + for (int i = 0; i < numByteFeatures; i++) { + String featureName = getFeatureName(i); + if (getValuesOf10.contains(featureName)) { + out.print("10 | " + featureName); + } else { + boolean found = false; + for (String match : getValuesOf5) { + if (featureName.matches(".*" + match)) { + out.print("5 | " + featureName); + found = true; + break; + } + } + if (!found) { + out.print("0 | " + featureName); + } + } + for (int v = 0, vmax = getNumberOfValues(i); v < vmax; v++) { + String val = getFeatureValueAsString(i, v); + out.print(" " + val); + } + out.print("\n"); + } + out.println("ShortValuedFeatureProcessors"); + for (int i = numByteFeatures; i < numShortFeatures; i++) { + String featureName = getFeatureName(i); + out.print("0 | " + featureName); + for (int v = 0, vmax = getNumberOfValues(i); v < vmax; v++) { + String val = getFeatureValueAsString(i, v); + out.print(" " + val); + } + out.print("\n"); + } + out.println("ContinuousFeatureProcessors"); + for (int i = numByteFeatures; i < numByteFeatures + numContinuousFeatures; i++) { + String featureName = getFeatureName(i); + out.println("0 linear | " + featureName); + } + out.flush(); + out.close(); + } + + /** + * Compares two feature vectors in terms of how many discrete features they have in common. WARNING: this assumes that the + * feature vectors are issued from the same FeatureDefinition; only the number of features is checked for compatibility. + * + * @param v1 + * A feature vector. + * @param v2 + * Another feature vector to compare v1 with. + * @return The number of common features. + */ + public static int diff(FeatureVector v1, FeatureVector v2) { + + int ret = 0; + + /* Byte valued features */ + if (v1.byteValuedDiscreteFeatures.length < v2.byteValuedDiscreteFeatures.length) { + throw new RuntimeException("v1 and v2 don't have the same number of byte-valued features: [" + + v1.byteValuedDiscreteFeatures.length + "] versus [" + v2.byteValuedDiscreteFeatures.length + "]."); + } + for (int i = 0; i < v1.byteValuedDiscreteFeatures.length; i++) { + if (v1.byteValuedDiscreteFeatures[i] == v2.byteValuedDiscreteFeatures[i]) + ret++; + } + + /* Short valued features */ + if (v1.shortValuedDiscreteFeatures.length < v2.shortValuedDiscreteFeatures.length) { + throw new RuntimeException("v1 and v2 don't have the same number of short-valued features: [" + + v1.shortValuedDiscreteFeatures.length + "] versus [" + v2.shortValuedDiscreteFeatures.length + "]."); + } + for (int i = 0; i < v1.shortValuedDiscreteFeatures.length; i++) { + if (v1.shortValuedDiscreteFeatures[i] == v2.shortValuedDiscreteFeatures[i]) + ret++; + } + + /* TODO: would checking float-valued features make sense ? (Code below.) */ + /* float valued features */ + /* + * if ( v1.continuousFeatures.length < v2.continuousFeatures.length ) { throw new RuntimeException( + * "v1 and v2 don't have the same number of continuous features: [" + v1.continuousFeatures.length + "] versus [" + + * v2.continuousFeatures.length + "]." ); } float epsilon = 1.0e-6f; float d = 0.0f; for ( int i = 0; i < + * v1.continuousFeatures.length; i++ ) { d = ( v1.continuousFeatures[i] > v2.continuousFeatures[i] ? + * (v1.continuousFeatures[i] - v2.continuousFeatures[i]) : (v2.continuousFeatures[i] - v1.continuousFeatures[i]) ); // => + * this avoids Math.abs() if ( d < epsilon ) ret++; } + */ + + return (ret); + } + +} diff --git a/marytts-unitselection/src/main/java/marytts/features/FeatureProcessorManager.java b/marytts-unitselection/src/main/java/marytts/features/FeatureProcessorManager.java new file mode 100644 index 0000000000..5441f80b73 --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/features/FeatureProcessorManager.java @@ -0,0 +1,344 @@ +/** + * Copyright 2006-2007 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.features; + +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; + +import marytts.exceptions.MaryConfigurationException; +import marytts.modules.acoustic.Model; +import marytts.modules.phonemiser.AllophoneSet; +import marytts.modules.synthesis.Voice; +import marytts.server.MaryProperties; +import marytts.util.MaryRuntimeUtils; +import marytts.util.MaryUtils; + +public class FeatureProcessorManager { + protected Map processors; + + protected Map phonefeatures2values; + + protected Locale locale; + + public FeatureProcessorManager(String localeString) throws MaryConfigurationException { + this(MaryUtils.string2locale(localeString)); + } + + public FeatureProcessorManager(Locale locale) throws MaryConfigurationException { + this.locale = locale; + setupGenericFeatureProcessors(); + + AllophoneSet allophoneSet = MaryRuntimeUtils.needAllophoneSet(MaryProperties.localePrefix(locale) + ".allophoneset"); + setupPhoneFeatureProcessors(allophoneSet, null, null, null); + } + + /** + * Constructor called from a Voice that has its own acoustic models + */ + public FeatureProcessorManager(Voice voice) throws MaryConfigurationException { + this(voice.getLocale()); + registerAcousticModels(voice); + } + + /** + * Create any additional feature processors for acoustic models. + * + * @param voice + */ + protected void registerAcousticModels(Voice voice) { + Map acousticModels = voice.getAcousticModels(); + if (acousticModels == null) { + return; + } + for (Model model : acousticModels.values()) { + // does this model predict a custom continuous feature...? + String modelFeatureName = model.getFeatureName(); + if (modelFeatureName != null && !listContinuousFeatureProcessorNames().contains(modelFeatureName)) { + // ...then add a generic featureProcessor for the custom feature: + String modelAttributeName = model.getTargetAttributeName(); + MaryFeatureProcessor featureProcessor = new MaryGenericFeatureProcessors.GenericContinuousFeature( + modelFeatureName, modelAttributeName); + addFeatureProcessor(featureProcessor); + } + } + } + + /** + * This constructor should not be used anymore. It contains hard-coded feature lists. Use + * {@link #FeatureProcessorManager(Locale)} instead. + */ + @Deprecated + public FeatureProcessorManager() { + setupGenericFeatureProcessors(); + } + + @Deprecated + protected void setupHardcodedPhoneFeatureValues() { + // Set up default values for phone features: + phonefeatures2values = new HashMap(); + // cplace: 0-n/a l-labial a-alveolar p-palatal b-labio_dental d-dental v-velar g-? + phonefeatures2values.put("cplace", new String[] { "0", "l", "a", "p", "b", "d", "v", "g" }); + // ctype: 0-n/a s-stop f-fricative a-affricative n-nasal l-liquid r-r + phonefeatures2values.put("ctype", new String[] { "0", "s", "f", "a", "n", "l", "r" }); + // cvox: 0=n/a +=on -=off + phonefeatures2values.put("cvox", new String[] { "0", "+", "-" }); + // vc: 0=n/a +=vowel -=consonant + phonefeatures2values.put("vc", new String[] { "0", "+", "-" }); + // vfront: 0-n/a 1-front 2-mid 3-back + phonefeatures2values.put("vfront", new String[] { "0", "1", "2", "3" }); + // vheight: 0-n/a 1-high 2-mid 3-low + phonefeatures2values.put("vheight", new String[] { "0", "1", "2", "3" }); + // vlng: 0-n/a s-short l-long d-dipthong a-schwa + phonefeatures2values.put("vlng", new String[] { "0", "s", "l", "d", "a" }); + // vrnd: 0=n/a +=on -=off + phonefeatures2values.put("vrnd", new String[] { "0", "+", "-" }); + } + + protected void setupGenericFeatureProcessors() { + processors = new TreeMap(); + + MaryGenericFeatureProcessors.TargetElementNavigator segment = new MaryGenericFeatureProcessors.SegmentNavigator(); + MaryGenericFeatureProcessors.TargetElementNavigator prevSegment = new MaryGenericFeatureProcessors.PrevSegmentNavigator(); + MaryGenericFeatureProcessors.TargetElementNavigator nextSegment = new MaryGenericFeatureProcessors.NextSegmentNavigator(); + MaryGenericFeatureProcessors.TargetElementNavigator syllable = new MaryGenericFeatureProcessors.SyllableNavigator(); + MaryGenericFeatureProcessors.TargetElementNavigator prevSyllable = new MaryGenericFeatureProcessors.PrevSyllableNavigator(); + MaryGenericFeatureProcessors.TargetElementNavigator nextSyllable = new MaryGenericFeatureProcessors.NextSyllableNavigator(); + MaryGenericFeatureProcessors.TargetElementNavigator nextNextSyllable = new MaryGenericFeatureProcessors.NextNextSyllableNavigator(); + MaryGenericFeatureProcessors.TargetElementNavigator lastWord = new MaryGenericFeatureProcessors.LastWordInSentenceNavigator(); + + addFeatureProcessor(new MaryGenericFeatureProcessors.Edge()); + addFeatureProcessor(new MaryGenericFeatureProcessors.HalfPhoneLeftRight()); + addFeatureProcessor(new MaryGenericFeatureProcessors.Accented("accented", syllable)); + addFeatureProcessor(new MaryGenericFeatureProcessors.Stressed("stressed", syllable)); + addFeatureProcessor(new MaryGenericFeatureProcessors.Stressed("prev_stressed", prevSyllable)); + addFeatureProcessor(new MaryGenericFeatureProcessors.Stressed("next_stressed", nextSyllable)); + addFeatureProcessor(new MaryGenericFeatureProcessors.WordNumSyls()); + addFeatureProcessor(new MaryGenericFeatureProcessors.PosInSyl()); + addFeatureProcessor(new MaryGenericFeatureProcessors.SylBreak("syl_break", syllable)); + addFeatureProcessor(new MaryGenericFeatureProcessors.SylBreak("prev_syl_break", prevSyllable)); + addFeatureProcessor(new MaryGenericFeatureProcessors.PositionType()); + addFeatureProcessor(new MaryGenericFeatureProcessors.BreakIndex()); + addFeatureProcessor(new MaryGenericFeatureProcessors.IsPause("prev_is_pause", prevSegment)); + addFeatureProcessor(new MaryGenericFeatureProcessors.IsPause("next_is_pause", nextSegment)); + addFeatureProcessor(new MaryGenericFeatureProcessors.TobiAccent("tobi_accent", syllable)); + addFeatureProcessor(new MaryGenericFeatureProcessors.TobiAccent("next_tobi_accent", nextSyllable)); + addFeatureProcessor(new MaryGenericFeatureProcessors.TobiAccent("nextnext_tobi_accent", nextNextSyllable)); + addFeatureProcessor(new MaryGenericFeatureProcessors.TobiEndtone("tobi_endtone", syllable)); + addFeatureProcessor(new MaryGenericFeatureProcessors.TobiEndtone("next_tobi_endtone", nextSyllable)); + addFeatureProcessor(new MaryGenericFeatureProcessors.TobiEndtone("nextnext_tobi_endtone", nextNextSyllable)); + addFeatureProcessor(new MaryGenericFeatureProcessors.WordPunc("sentence_punc", lastWord)); + addFeatureProcessor(new MaryGenericFeatureProcessors.SylsFromPhraseStart()); + addFeatureProcessor(new MaryGenericFeatureProcessors.SylsFromPhraseEnd()); + addFeatureProcessor(new MaryGenericFeatureProcessors.StressedSylsFromPhraseStart()); + addFeatureProcessor(new MaryGenericFeatureProcessors.StressedSylsFromPhraseEnd()); + addFeatureProcessor(new MaryGenericFeatureProcessors.AccentedSylsFromPhraseStart()); + addFeatureProcessor(new MaryGenericFeatureProcessors.AccentedSylsFromPhraseEnd()); + addFeatureProcessor(new MaryGenericFeatureProcessors.SylsFromPrevStressed()); + addFeatureProcessor(new MaryGenericFeatureProcessors.SylsToNextStressed()); + addFeatureProcessor(new MaryGenericFeatureProcessors.SylsFromPrevAccent()); + addFeatureProcessor(new MaryGenericFeatureProcessors.SylsToNextAccent()); + addFeatureProcessor(new MaryGenericFeatureProcessors.WordNumSegs()); + addFeatureProcessor(new MaryGenericFeatureProcessors.SegsFromSylStart()); + addFeatureProcessor(new MaryGenericFeatureProcessors.SegsFromSylEnd()); + addFeatureProcessor(new MaryGenericFeatureProcessors.SylNumSegs()); + addFeatureProcessor(new MaryGenericFeatureProcessors.SentenceNumPhrases()); + addFeatureProcessor(new MaryGenericFeatureProcessors.SentenceNumWords()); + addFeatureProcessor(new MaryGenericFeatureProcessors.PhraseNumWords()); + addFeatureProcessor(new MaryGenericFeatureProcessors.PhraseNumSyls()); + addFeatureProcessor(new MaryGenericFeatureProcessors.SegsFromWordStart()); + addFeatureProcessor(new MaryGenericFeatureProcessors.SegsFromWordEnd()); + addFeatureProcessor(new MaryGenericFeatureProcessors.SylsFromWordStart()); + addFeatureProcessor(new MaryGenericFeatureProcessors.SylsFromWordEnd()); + addFeatureProcessor(new MaryGenericFeatureProcessors.WordsFromPhraseStart()); + addFeatureProcessor(new MaryGenericFeatureProcessors.WordsFromPhraseEnd()); + addFeatureProcessor(new MaryGenericFeatureProcessors.WordsFromSentenceStart()); + addFeatureProcessor(new MaryGenericFeatureProcessors.WordsFromSentenceEnd()); + addFeatureProcessor(new MaryGenericFeatureProcessors.PhrasesFromSentenceStart()); + addFeatureProcessor(new MaryGenericFeatureProcessors.PhrasesFromSentenceEnd()); + addFeatureProcessor(new MaryGenericFeatureProcessors.NextAccent()); + addFeatureProcessor(new MaryGenericFeatureProcessors.PrevAccent()); + addFeatureProcessor(new MaryGenericFeatureProcessors.PhraseEndtone()); + addFeatureProcessor(new MaryGenericFeatureProcessors.PrevPhraseEndtone()); + addFeatureProcessor(new MaryGenericFeatureProcessors.PrevPunctuation()); + addFeatureProcessor(new MaryGenericFeatureProcessors.NextPunctuation()); + addFeatureProcessor(new MaryGenericFeatureProcessors.WordsFromPrevPunctuation()); + addFeatureProcessor(new MaryGenericFeatureProcessors.WordsToNextPunctuation()); + addFeatureProcessor(new MaryGenericFeatureProcessors.Selection_Prosody(syllable)); + addFeatureProcessor(new MaryGenericFeatureProcessors.Style()); + + addFeatureProcessor(new MaryGenericFeatureProcessors.UnitDuration()); + addFeatureProcessor(new MaryGenericFeatureProcessors.UnitLogF0()); + addFeatureProcessor(new MaryGenericFeatureProcessors.UnitLogF0Delta()); + } + + /** + * Get the locale for this feature processor manager. Locale-specific subclasses must override this method to return the + * respective locale. This base class returns null to indicate that the feature processor manager can be used as a fallback + * for any locale. + * + * @return + */ + public Locale getLocale() { + return locale; + } + + /** + * Provide a space-separated list of the feature names for all the feature processors known to this feature processor manager. + * The list is unsorted except that byte-valued feature processors come first, followed by short-valued feature processors, + * followed by continuous feature processors. + * + * @return + */ + public String listFeatureProcessorNames() { + String bytes = listByteValuedFeatureProcessorNames(); + String shorts = listShortValuedFeatureProcessorNames(); + String conts = listContinuousFeatureProcessorNames(); + StringBuilder sb = new StringBuilder(bytes.length() + shorts.length() + conts.length() + 2); + sb.append(bytes); + if (bytes.length() > 0 && shorts.length() > 0) { + sb.append(" "); + } + sb.append(shorts); + if (conts.length() > 0) { + sb.append(" "); + } + sb.append(conts); + return sb.toString(); + } + + public String listByteValuedFeatureProcessorNames() { + StringBuilder sb = new StringBuilder(); + for (String name : processors.keySet()) { + MaryFeatureProcessor fp = processors.get(name); + if (fp instanceof ByteValuedFeatureProcessor) { + if (sb.length() > 0) + sb.append(" "); + sb.append(name); + } + } + return sb.toString(); + } + + public String listShortValuedFeatureProcessorNames() { + StringBuilder sb = new StringBuilder(); + for (String name : processors.keySet()) { + MaryFeatureProcessor fp = processors.get(name); + if (fp instanceof ShortValuedFeatureProcessor) { + if (sb.length() > 0) + sb.append(" "); + sb.append(name); + } + } + return sb.toString(); + } + + public String listContinuousFeatureProcessorNames() { + StringBuilder sb = new StringBuilder(); + for (String name : processors.keySet()) { + MaryFeatureProcessor fp = processors.get(name); + if (fp instanceof ContinuousFeatureProcessor) { + if (sb.length() > 0) + sb.append(" "); + sb.append(name); + } + } + return sb.toString(); + } + + protected void addFeatureProcessor(MaryFeatureProcessor fp) { + processors.put(fp.getName(), fp); + } + + public MaryFeatureProcessor getFeatureProcessor(String name) { + return processors.get(name); + } + + /** + * Set up phone feature processors based on phoneset. + * + * @param phoneset + * the AllophoneSet used for the current locale. + * @param phoneValues + * optional. If null, will query phoneset. + * @param pauseSymbol + * optional. If null, will query phoneset. + * @param featuresToValues + * map listing the possible values for each feature. Optional. If null, will query phoneset. + */ + protected void setupPhoneFeatureProcessors(AllophoneSet phoneset, String[] phoneValues, String pauseSymbol, + Map featuresToValues) { + MaryGenericFeatureProcessors.TargetElementNavigator segment = new MaryGenericFeatureProcessors.SegmentNavigator(); + + if (phoneValues == null) { + String[] pValues = (String[]) phoneset.getAllophoneNames().toArray(new String[0]); + phoneValues = new String[pValues.length + 1]; + phoneValues[0] = "0"; + System.arraycopy(pValues, 0, phoneValues, 1, pValues.length); + } + if (pauseSymbol == null) { + pauseSymbol = phoneset.getSilence().name(); + } + addFeatureProcessor(new MaryLanguageFeatureProcessors.Phone("phone", phoneValues, pauseSymbol, segment)); + addFeatureProcessor(new MaryLanguageFeatureProcessors.HalfPhoneUnitName(phoneValues, pauseSymbol)); + addFeatureProcessor(new MaryLanguageFeatureProcessors.SegOnsetCoda(phoneset)); + // Phone features: + Set featureNames; + if (featuresToValues != null) + featureNames = featuresToValues.keySet(); + else + featureNames = phoneset.getPhoneFeatures(); + for (String feature : featureNames) { + String[] values; + if (featuresToValues != null) + values = featuresToValues.get(feature); + else + values = phoneset.getPossibleFeatureValues(feature); + addFeatureProcessor(new MaryLanguageFeatureProcessors.PhoneFeature(phoneset, "ph_" + feature, feature, values, + pauseSymbol, segment)); + } + + Map segments = new HashMap(); + + segments.put("prev", new MaryGenericFeatureProcessors.PrevSegmentNavigator()); + segments.put("prev_prev", new MaryGenericFeatureProcessors.PrevPrevSegmentNavigator()); + segments.put("next", new MaryGenericFeatureProcessors.NextSegmentNavigator()); + segments.put("next_next", new MaryGenericFeatureProcessors.NextNextSegmentNavigator()); + + for (String position : segments.keySet()) { + MaryGenericFeatureProcessors.TargetElementNavigator navi = segments.get(position); + addFeatureProcessor(new MaryLanguageFeatureProcessors.Phone(position + "_phone", phoneValues, pauseSymbol, navi)); + // Phone features: + for (String feature : featureNames) { + String[] values; + if (featuresToValues != null) + values = featuresToValues.get(feature); + else + values = phoneset.getPossibleFeatureValues(feature); + addFeatureProcessor(new MaryLanguageFeatureProcessors.PhoneFeature(phoneset, position + "_" + feature, feature, + values, pauseSymbol, navi)); + } + } + + } + +} diff --git a/marytts-unitselection/src/main/java/marytts/features/FeatureRegistry.java b/marytts-unitselection/src/main/java/marytts/features/FeatureRegistry.java new file mode 100644 index 0000000000..425e38f875 --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/features/FeatureRegistry.java @@ -0,0 +1,241 @@ +/** + * Copyright 2009 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ + +package marytts.features; + +import java.util.Collection; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; +import java.util.StringTokenizer; +import java.util.TreeSet; + +import marytts.modules.synthesis.Voice; + +import org.apache.commons.collections.map.MultiKeyMap; + +/** + * @author marc + * + */ +public class FeatureRegistry { + /** + * No instances of this class. + */ + private FeatureRegistry() { + } + + private static Map managersByLocale = new HashMap(); + private static Map managersByVoice = new HashMap(); + private static FeatureProcessorManager fallbackManager = null; + private static MultiKeyMap/* */computers = new MultiKeyMap(); + + /** + * Set the given feature processor manager as the one to use for the given locale. + * + * @param locale + * @param mgr + */ + public static void setFeatureProcessorManager(Locale locale, FeatureProcessorManager mgr) { + managersByLocale.put(locale, mgr); + } + + /** + * Set the given feature processor manager as the one to use when no voice- or locale-specific feature processor manager can + * be found. + * + * @param mgr + */ + public static void setFallbackFeatureProcessorManager(FeatureProcessorManager mgr) { + fallbackManager = mgr; + } + + /** + * Set the given feature processor manager as the one to use for the given voice. + * + * @param voice + * @param mgr + */ + public static void setFeatureProcessorManager(Voice voice, FeatureProcessorManager mgr) { + managersByVoice.put(voice, mgr); + } + + /** + * Get the feature processor manager associated with the given voice, if any. + * + * @param voice + * @return the feature processor manager, or null if there is no voice-specific feature processor manager. + */ + public static FeatureProcessorManager getFeatureProcessorManager(Voice voice) { + return managersByVoice.get(voice); + } + + /** + * Get the feature processor manager associated with the given locale, if any. + * + * @param locale + * @return the feature processor manager, or null if there is no locale-specific feature processor manager. + */ + public static FeatureProcessorManager getFeatureProcessorManager(Locale locale) { + FeatureProcessorManager m = managersByLocale.get(locale); + if (m != null) + return m; + // Maybe locale is language_COUNTRY, so look up by language also: + Locale lang = new Locale(locale.getLanguage()); + return managersByLocale.get(lang); + } + + /** + * Get the fallback feature processor manager which should be used if there is no voice- or locale-specific feature processor + * manager. + * + * @return + */ + public static FeatureProcessorManager getFallbackFeatureProcessorManager() { + return fallbackManager; + } + + /** + * For the given voice, return the best feature manager. That is either the voice-specific feature manager, if any, or the + * locale-specific feature manager, if any, or the language-specific feature manager, if any, or the fallback feature manager. + * + * @param voice + * @return a feature processor manager object. If this returns null, something is broken. + */ + public static FeatureProcessorManager determineBestFeatureProcessorManager(Voice voice) { + FeatureProcessorManager mgr = getFeatureProcessorManager(voice); + if (mgr == null) { + mgr = determineBestFeatureProcessorManager(voice.getLocale()); + } + return mgr; + } + + /** + * For the given locale, return the best feature manager. That is either the locale-specific feature manager, if any, or the + * language-specific feature manager, if any, or the fallback feature manager. + * + * @param locale + * @return a feature processor manager object. If this returns null, something is broken. + */ + public static FeatureProcessorManager determineBestFeatureProcessorManager(Locale locale) { + FeatureProcessorManager mgr = getFeatureProcessorManager(locale); + // Locale can have been en_US etc, i.e. language + country; let's try + // language only as well. + if (mgr == null) { + Locale lang = new Locale(locale.getLanguage()); + mgr = getFeatureProcessorManager(lang); + } + if (mgr == null) { + mgr = getFallbackFeatureProcessorManager(); + } + assert mgr != null; + return mgr; + } + + public static Collection getSupportedLocales() { + Collection locales = new TreeSet(new Comparator() { + public int compare(Locale o1, Locale o2) { + if (o1 == null) { + if (o2 == null) + return 0; + return -1; + } + if (o2 == null) { + return 1; + } + return o1.toString().compareTo(o2.toString()); + } + }); + locales.addAll(managersByLocale.keySet()); + return locales; + } + + /** + * Obtain a TargetFeatureComputer that knows how to compute features for a Target using the given set of feature processor + * names. These names must be known to the given Feature processor manager. + * + * @param mgr + * @param features + * a String containing the names of the feature processors to use, separated by white space, and in the right order + * (byte-valued discrete feature processors first, then short-valued, then continuous). If features is null, use + * all available features processors. + * @return a target feature computer + * @throws IllegalArgumentException + * if one of the features is not known to the manager + */ + public static TargetFeatureComputer getTargetFeatureComputer(FeatureProcessorManager mgr, String features) { + if (features == null) { + features = mgr.listFeatureProcessorNames(); + } else { + // verify that each feature is known to the mgr + StringTokenizer st = new StringTokenizer(features); + while (st.hasMoreTokens()) { + String feature = st.nextToken(); + if (mgr.getFeatureProcessor(feature) == null) { + throw new IllegalArgumentException("Feature processor manager '" + mgr.getClass().toString() + + "' does not know the feature '" + feature + "'"); + } + + } + } + TargetFeatureComputer tfc = (TargetFeatureComputer) computers.get(mgr, features); + if (tfc == null) { + tfc = new TargetFeatureComputer(mgr, features); + } + return tfc; + } + + /** + * Convenience method for getting a suitable target feature computer for the given locale and list of features. A feature + * processor for the given locale is looked up using {@link #getFeatureProcessorManager(Locale)} or, if that fails, using + * {@link #getFallbackFeatureProcessorManager()}. + * + * @see #getTargetFeatureComputer(FeatureProcessorManager, String) + * @param locale + * @param features + * a String containing the names of the feature processors to use, separated by white space, and in the right order + * (byte-valued discrete feature processors first, then short-valued, then continuous) + * @return a target feature computer + */ + public static TargetFeatureComputer getTargetFeatureComputer(Locale locale, String features) { + FeatureProcessorManager mgr = determineBestFeatureProcessorManager(locale); + return getTargetFeatureComputer(mgr, features); + } + + /** + * Convenience method for getting a suitable target feature computer for the given voice and list of features. A feature + * processor for the given voice is looked up using {@link #getFeatureProcessorManager(Voice)} or, if that fails, using + * {@link #getFeatureProcessorManager(Locale)} using the voice locale or, if that also fails, using + * {@link #getFallbackFeatureProcessorManager()}. + * + * @see #getTargetFeatureComputer(FeatureProcessorManager, String) + * @param locale + * @param features + * a String containing the names of the feature processors to use, separated by white space, and in the right order + * (byte-valued discrete feature processors first, then short-valued, then continuous) + * @return a target feature computer + */ + public static TargetFeatureComputer getTargetFeatureComputer(Voice voice, String features) { + FeatureProcessorManager mgr = determineBestFeatureProcessorManager(voice); + return getTargetFeatureComputer(mgr, features); + } + +} diff --git a/marytts-unitselection/src/main/java/marytts/features/FeatureVector.java b/marytts-unitselection/src/main/java/marytts/features/FeatureVector.java new file mode 100644 index 0000000000..a3e4d0c624 --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/features/FeatureVector.java @@ -0,0 +1,326 @@ +/** + * Copyright 2000-2009 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.features; + +import java.io.DataOutput; +import java.io.IOException; + +/** + * Compact representation of a list of byte-valued, short-valued and float-valued (continuous) features. In the user interface, + * features are identified through one index number, covering all three types of features. For example, a feature vector + * consisting of three bytes, four shorts and two floats will have nine features. Use getByteFeature(), getShortFeature() and + * getContinuousFeature() to access the features as primitives; note that you have to respect the index restrictions, i.e. in the + * example, calling getShortFeature(6) would be OK, but getShortFeature(2) or getShortFeature(8) would throw an + * IndexOutOfBoundsException. Use isShortFeature(i) to test whether a given feature is a short feature. Alternatively, you can use + * an Object interface to access all features in a uniform way: getFeature(i) will return a Number object for all valid indexes. + * + * @author Marc Schröder + * + */ +public class FeatureVector { + public enum FeatureType { + byteValued, shortValued, floatValued + }; + + public final int unitIndex; + public final byte[] byteValuedDiscreteFeatures; + public final short[] shortValuedDiscreteFeatures; + public final float[] continuousFeatures; + + public FeatureVector(byte[] byteValuedDiscreteFeatures, short[] shortValuedDiscreteFeatures, float[] continuousFeatures, + int setUnitIndex) { + this.byteValuedDiscreteFeatures = byteValuedDiscreteFeatures; + this.shortValuedDiscreteFeatures = shortValuedDiscreteFeatures; + this.continuousFeatures = continuousFeatures; + if (setUnitIndex < 0) { + throw new RuntimeException("The unit index can't be negative or null when instanciating a new feature vector."); + } + this.unitIndex = setUnitIndex; + } + + /** + * Is this an edge vector? + * + * @return isEdge + */ + public boolean isEdgeVector(int edgeIndex) { + String edgeValue = getFeature(edgeIndex).toString(); + return (!edgeValue.equals(FeatureDefinition.NULLVALUE)); + } + + public FeatureType getFeatureType(int featureIndex) { + FeatureType t = null; + if (featureIndex < 0 + || featureIndex >= byteValuedDiscreteFeatures.length + shortValuedDiscreteFeatures.length + + continuousFeatures.length) { + throw new IllegalArgumentException("Index " + featureIndex + " is out of range [0, " + getLength() + "["); + } + if (featureIndex < byteValuedDiscreteFeatures.length) { + t = FeatureType.byteValued; + } else if (featureIndex < byteValuedDiscreteFeatures.length + shortValuedDiscreteFeatures.length) { + t = FeatureType.shortValued; + } else if (featureIndex < byteValuedDiscreteFeatures.length + shortValuedDiscreteFeatures.length + + continuousFeatures.length) { + t = FeatureType.floatValued; + } + return t; + } + + /** + * Get the total number of features in this feature vector. + * + * @return the number of features, irrespective of their type + */ + public int getLength() { + return byteValuedDiscreteFeatures.length + shortValuedDiscreteFeatures.length + continuousFeatures.length; + } + + /** + * Get the index of the unit to which the current feature vector applies. + * + * @return The related unit index. + */ + public int getUnitIndex() { + return (unitIndex); + } + + /** + * The number of byte features in this feature vector. + * + * @return a number of byte features, possibly 0. + */ + public int getNumberOfByteFeatures() { + return byteValuedDiscreteFeatures.length; + } + + /** + * The number of short features in this feature vector. + * + * @return a number of short features, possibly 0. + */ + public int getNumberOfShortFeatures() { + return shortValuedDiscreteFeatures.length; + } + + /** + * The number of continuous features in this feature vector. + * + * @return a number of continuous features, possibly 0. + */ + public int getNumberOfContinuousFeatures() { + return continuousFeatures.length; + } + + /** + * A uniform way to access features in this feature vector. + * + * @param index + * a feature index between 0 and getLength()-1 + * @return a Number object, which will be a Byte, a Short or a Float depending on the type of the feature with the given index + * number. + */ + public Number getFeature(int index) { + if (index < byteValuedDiscreteFeatures.length) + return new Byte(byteValuedDiscreteFeatures[index]); + index -= byteValuedDiscreteFeatures.length; + if (index < shortValuedDiscreteFeatures.length) + return new Short(shortValuedDiscreteFeatures[index]); + index -= shortValuedDiscreteFeatures.length; + if (index < continuousFeatures.length) + return new Float(continuousFeatures[index]); + throw new IndexOutOfBoundsException(); + } + + /** + * A wrapper to getFeature(), to get the result as an int value, e.g., for subsequent array indexing. + * + * @param index + * A feature index between 0 and getLength()-1. + * @return The feature value, as an int. + * + * @see FeatureVector#getFeature(int) + */ + public int getFeatureAsInt(int index) { + return (getFeature(index).intValue()); + } + + /** + * A wrapper to getFeature(), to get the result as an String value, e.g., for subsequent System.out output. + * + * @param index + * A feature index between 0 and getLength()-1. + * @param feaDef + * A FeatureDefinition object allowing to decode the feature value. + * @return The feature value, as a String. + * + * @see FeatureVector#getFeature(int) + */ + public String getFeatureAsString(int index, FeatureDefinition feaDef) { + if (index < byteValuedDiscreteFeatures.length) + return feaDef.getFeatureValueAsString(index, byteValuedDiscreteFeatures[index]); + throw new IndexOutOfBoundsException(); + } + + /** + * An efficient way to access byte-valued features in this feature vector. + * + * @param index + * the index number of the byte-valued feature in this feature vector. + * @return the byte value of the feature with the given index. + * @throws IndexOutOfBoundsException + * if index<0 or index >= getNumberOfByteFeatures(). + * @see #getNumberOfByteFeatures() + * @see #isByteFeature() + */ + public final byte getByteFeature(int index) { + if (index < 0 || index >= byteValuedDiscreteFeatures.length) { + throw new IndexOutOfBoundsException(index + " is not between 0 and " + byteValuedDiscreteFeatures.length); + } + return byteValuedDiscreteFeatures[index]; + } + + /** + * An efficient way to access short-valued features in this feature vector. + * + * @param index + * the index number of the short-valued feature in this feature vector. + * @return the short value of the feature with the given index. + * @throws IndexOutOfBoundsException + * if index= getNumberOfByteFeatures()+getNumberOfShortFeatures(). + * @see #getNumberOfByteFeatures() + * @see #getNumberOfShortFeatures() + * @see #isShortFeature() + */ + public final short getShortFeature(int index) { + return shortValuedDiscreteFeatures[index - byteValuedDiscreteFeatures.length]; + } + + /** + * An efficient way to access continuous features in this feature vector. + * + * @param index + * the index number of the continuous feature in this feature vector. + * @return the float value of the feature with the given index. + * @throws IndexOutOfBoundsException + * if index= getLength(). + * @see #getNumberOfByteFeatures() + * @see #getNumberOfShortFeatures() + * @see #getNumberOfContinuousFeatures() + * @see #getLength() + * @see #isContinuousFeature() + */ + public final float getContinuousFeature(int index) { + return continuousFeatures[index - byteValuedDiscreteFeatures.length - shortValuedDiscreteFeatures.length]; + } + + /** + * Test whether the feature with the given index number is a byte feature. + * + * @param index + * @return This will return true exactly if getByteFeature(index) would return a value without throwing an exception, i.e. if + * index>=0 and index < getNumberOfByteFeatures(). + */ + public boolean isByteFeature(int index) { + return 0 <= index && index < byteValuedDiscreteFeatures.length; + } + + /** + * Test whether the feature with the given index number is a short feature. + * + * @param index + * @return This will return true exactly if getShortFeature(index) would return a value without throwing an exception, i.e. if + * index>=getNumberOfByteFeatures() and index < getNumberOfByteFeatures()+getNumberOfShortFeatures(). + */ + public boolean isShortFeature(int index) { + return byteValuedDiscreteFeatures.length <= index + && index < byteValuedDiscreteFeatures.length + shortValuedDiscreteFeatures.length; + } + + /** + * Test whether the feature with the given index number is a continuous feature. + * + * @param index + * @return This will return true exactly if getContinuousFeature(index) would return a value without throwing an exception, + * i.e. if index>=getNumberOfByteFeatures()+getNumberOfShortFeatures() and index < getLength(). + */ + public boolean isContinuousFeature(int index) { + return byteValuedDiscreteFeatures.length + shortValuedDiscreteFeatures.length <= index + && index < byteValuedDiscreteFeatures.length + shortValuedDiscreteFeatures.length + continuousFeatures.length; + } + + public byte[] getByteValuedDiscreteFeatures() { + return byteValuedDiscreteFeatures; + } + + public short[] getShortValuedDiscreteFeatures() { + return shortValuedDiscreteFeatures; + } + + public float[] getContinuousFeatures() { + return continuousFeatures; + } + + /** + * Write a binary representation of this feature vector to the given data output. + * + * @param out + * the DataOutputStream or RandomAccessFile in which to write the binary representation of the feature vector. + * @return + */ + public void writeTo(DataOutput out) throws IOException { + if (byteValuedDiscreteFeatures != null) { + out.write(byteValuedDiscreteFeatures); + } + if (shortValuedDiscreteFeatures != null) { + for (int i = 0; i < shortValuedDiscreteFeatures.length; i++) { + out.writeShort(shortValuedDiscreteFeatures[i]); + } + } + if (continuousFeatures != null) { + for (int i = 0; i < continuousFeatures.length; i++) { + out.writeFloat(continuousFeatures[i]); + } + } + } + + /** + * Return a string representation of this set of target features; feature values separated by spaces. + */ + public String toString() { + StringBuilder out = new StringBuilder(); + for (int i = 0; i < byteValuedDiscreteFeatures.length; i++) { + if (out.length() > 0) + out.append(" "); + out.append((int) byteValuedDiscreteFeatures[i]); + } + for (int i = 0; i < shortValuedDiscreteFeatures.length; i++) { + if (out.length() > 0) + out.append(" "); + out.append((int) shortValuedDiscreteFeatures[i]); + } + for (int i = 0; i < continuousFeatures.length; i++) { + if (out.length() > 0) + out.append(" "); + out.append(continuousFeatures[i]); + } + return out.toString(); + } + +} diff --git a/marytts-unitselection/src/main/java/marytts/features/HalfPhoneTarget.java b/marytts-unitselection/src/main/java/marytts/features/HalfPhoneTarget.java new file mode 100644 index 0000000000..8e29ea8d3f --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/features/HalfPhoneTarget.java @@ -0,0 +1,60 @@ +/** + * Copyright 2006 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.features; + +import org.w3c.dom.Element; + +public class HalfPhoneTarget extends Target { + protected boolean isLeftHalf; + + /** + * Create a target associated to the given segment item. + * + * @param name + * a name for the target, which may or may not coincide with the segment name. + * @param item + * the phone segment item in the Utterance structure, to be associated to this target + * @param isLeftHalf + * true if this target represents the left half of the phone, false if it represents the right half of the phone + */ + public HalfPhoneTarget(String name, Element maryxmlElement, boolean isLeftHalf) { + super(name, maryxmlElement); + this.isLeftHalf = isLeftHalf; + } + + /** + * Is this target the left half of a phone? + * + * @return + */ + public boolean isLeftHalf() { + return isLeftHalf; + } + + /** + * Is this target the right half of a phone? + * + * @return + */ + public boolean isRightHalf() { + return !isLeftHalf; + } + +} diff --git a/marytts-unitselection/src/main/java/marytts/features/MaryFeatureProcessor.java b/marytts-unitselection/src/main/java/marytts/features/MaryFeatureProcessor.java new file mode 100644 index 0000000000..18a61c1338 --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/features/MaryFeatureProcessor.java @@ -0,0 +1,40 @@ +/** + * Portions Copyright 2006 DFKI GmbH. + * Portions Copyright 2001 Sun Microsystems, Inc. + * Portions Copyright 1999-2001 Language Technologies Institute, + * Carnegie Mellon University. + * All Rights Reserved. Use is subject to license terms. + * + * Permission is hereby granted, free of charge, to use and distribute + * this software and its documentation without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of this work, and to + * permit persons to whom this work is furnished to do so, subject to + * the following conditions: + * + * 1. The code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * 2. Any modifications must be clearly marked as such. + * 3. Original authors' names are not deleted. + * 4. The authors' names are not used to endorse or promote products + * derived from this software without specific prior written + * permission. + * + * DFKI GMBH AND THE CONTRIBUTORS TO THIS WORK DISCLAIM ALL WARRANTIES WITH + * REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL DFKI GMBH NOR THE + * CONTRIBUTORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL + * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR + * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS + * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF + * THIS SOFTWARE. + */ +package marytts.features; + +/** + * Performs a specific type of processing on an item and returns an object. + */ +public interface MaryFeatureProcessor { + + public String getName(); +} diff --git a/marytts-unitselection/src/main/java/marytts/features/MaryGenericFeatureProcessors.java b/marytts-unitselection/src/main/java/marytts/features/MaryGenericFeatureProcessors.java new file mode 100644 index 0000000000..ba8dc4be0b --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/features/MaryGenericFeatureProcessors.java @@ -0,0 +1,3055 @@ +/** + * Portions Copyright 2006-2007 DFKI GmbH. + * Portions Copyright 2001 Sun Microsystems, Inc. + * Portions Copyright 1999-2001 Language Technologies Institute, + * Carnegie Mellon University. + * All Rights Reserved. Use is subject to license terms. + * + * Permission is hereby granted, free of charge, to use and distribute + * this software and its documentation without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of this work, and to + * permit persons to whom this work is furnished to do so, subject to + * the following conditions: + * + * 1. The code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * 2. Any modifications must be clearly marked as such. + * 3. Original authors' names are not deleted. + * 4. The authors' names are not used to endorse or promote products + * derived from this software without specific prior written + * permission. + * + * DFKI GMBH AND THE CONTRIBUTORS TO THIS WORK DISCLAIM ALL WARRANTIES WITH + * REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL DFKI GMBH NOR THE + * CONTRIBUTORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL + * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR + * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS + * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF + * THIS SOFTWARE. + */ + +package marytts.features; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; +import java.util.StringTokenizer; + +import marytts.datatypes.MaryXML; +import marytts.util.dom.MaryDomUtils; +import marytts.util.string.ByteStringTranslator; + +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.traversal.TreeWalker; + +/** + * A collection of feature processors that operate on Target objects. + * + * @author schroed + * + */ +public class MaryGenericFeatureProcessors { + /** + * Navigate from a target to an item. Classes implementing this interface will retrieve meaningful items given the target. + * + * @author Marc Schröder + */ + public static interface TargetElementNavigator { + /** + * Given the target, retrieve an XML Element. + * + * @param target + * @return an item selected according to this navigator, or null if there is no such item. + */ + public Element getElement(Target target); + } + + /** + * Retrieve the segment belonging to this target. + * + * @author Marc Schröder + * + */ + public static class SegmentNavigator implements TargetElementNavigator { + public Element getElement(Target target) { + return target.getMaryxmlElement(); + } + } + + /** + * Retrieve the segment preceding the segment which belongs to this target. + * + * @author Marc Schröder + * + */ + public static class PrevSegmentNavigator implements TargetElementNavigator { + public Element getElement(Target target) { + Element segment = target.getMaryxmlElement(); + if (segment == null) + return null; + Element sentence = (Element) MaryDomUtils.getAncestor(segment, MaryXML.SENTENCE); + if (sentence == null) + return null; + TreeWalker tw = MaryDomUtils.createTreeWalker(sentence, MaryXML.PHONE, MaryXML.BOUNDARY); + tw.setCurrentNode(segment); + Element previous = (Element) tw.previousNode(); + return previous; + } + } + + /** + * Retrieve the segment two before the segment which belongs to this target. + * + * @author Marc Schröder + * + */ + public static class PrevPrevSegmentNavigator implements TargetElementNavigator { + public Element getElement(Target target) { + Element segment = target.getMaryxmlElement(); + if (segment == null) + return null; + Element sentence = (Element) MaryDomUtils.getAncestor(segment, MaryXML.SENTENCE); + if (sentence == null) + return null; + TreeWalker tw = MaryDomUtils.createTreeWalker(sentence, MaryXML.PHONE, MaryXML.BOUNDARY); + tw.setCurrentNode(segment); + Element previous = (Element) tw.previousNode(); + Element pp = (Element) tw.previousNode(); + return pp; + } + } + + /** + * Retrieve the segment following the segment which belongs to this target. + * + * @author Marc Schröder + * + */ + public static class NextSegmentNavigator implements TargetElementNavigator { + public Element getElement(Target target) { + Element segment = target.getMaryxmlElement(); + if (segment == null) + return null; + Element sentence = (Element) MaryDomUtils.getAncestor(segment, MaryXML.SENTENCE); + if (sentence == null) + return null; + TreeWalker tw = MaryDomUtils.createTreeWalker(sentence, MaryXML.PHONE, MaryXML.BOUNDARY); + tw.setCurrentNode(segment); + Element next = (Element) tw.nextNode(); + return next; + } + } + + /** + * Retrieve the segment two after the segment which belongs to this target. + * + * @author Marc Schröder + * + */ + public static class NextNextSegmentNavigator implements TargetElementNavigator { + public Element getElement(Target target) { + Element segment = target.getMaryxmlElement(); + if (segment == null) + return null; + Element sentence = (Element) MaryDomUtils.getAncestor(segment, MaryXML.SENTENCE); + if (sentence == null) + return null; + TreeWalker tw = MaryDomUtils.createTreeWalker(sentence, MaryXML.PHONE, MaryXML.BOUNDARY); + tw.setCurrentNode(segment); + Element next = (Element) tw.nextNode(); + Element nn = (Element) tw.nextNode(); + return nn; + } + } + + /** + * Retrieve the first segment in the word to which this target belongs. + * + */ + public static class FirstSegmentInWordNavigator implements TargetElementNavigator { + public Element getElement(Target target) { + Element segment = target.getMaryxmlElement(); + if (segment == null) + return null; + Element word = (Element) MaryDomUtils.getAncestor(segment, MaryXML.TOKEN); + if (word == null) + return null; + TreeWalker tw = MaryDomUtils.createTreeWalker(word, MaryXML.PHONE); + Element first = (Element) tw.firstChild(); + if (first != null) { + assert first.getTagName().equals(MaryXML.PHONE) : "Unexpected tag name: expected " + MaryXML.PHONE + ", got " + + first.getTagName(); + } + return first; + } + } + + /** + * Retrieve the last segment in the word to which this target belongs. + * + */ + public static class LastSegmentInWordNavigator implements TargetElementNavigator { + public Element getElement(Target target) { + Element segment = target.getMaryxmlElement(); + if (segment == null) + return null; + Element word = (Element) MaryDomUtils.getAncestor(segment, MaryXML.TOKEN); + if (word == null) + return null; + TreeWalker tw = MaryDomUtils.createTreeWalker(word, MaryXML.PHONE); + Element last = (Element) tw.lastChild(); + if (last != null) { + assert last.getTagName().equals(MaryXML.PHONE) : "Unexpected tag name: expected " + MaryXML.PHONE + ", got " + + last.getTagName(); + } + return last; + } + } + + /** + * Retrieve the first syllable in the word to which this target belongs. + * + */ + public static class FirstSyllableInWordNavigator implements TargetElementNavigator { + public Element getElement(Target target) { + Element segment = target.getMaryxmlElement(); + if (segment == null) + return null; + Element word = (Element) MaryDomUtils.getAncestor(segment, MaryXML.TOKEN); + if (word == null) + return null; + TreeWalker tw = MaryDomUtils.createTreeWalker(word, MaryXML.SYLLABLE); + Element first = (Element) tw.firstChild(); + if (first != null) { + assert first.getTagName().equals(MaryXML.SYLLABLE) : "Unexpected tag name: expected " + MaryXML.SYLLABLE + + ", got " + first.getTagName(); + } + return first; + } + } + + /** + * Retrieve the last syllable in the word to which this target belongs. + * + */ + public static class LastSyllableInWordNavigator implements TargetElementNavigator { + public Element getElement(Target target) { + Element segment = target.getMaryxmlElement(); + if (segment == null) + return null; + Element word = (Element) MaryDomUtils.getAncestor(segment, MaryXML.TOKEN); + if (word == null) + return null; + TreeWalker tw = MaryDomUtils.createTreeWalker(word, MaryXML.SYLLABLE); + Element last = (Element) tw.lastChild(); + if (last != null) { + assert last.getTagName().equals(MaryXML.SYLLABLE) : "Unexpected tag name: expected " + MaryXML.SYLLABLE + + ", got " + last.getTagName(); + } + return last; + } + } + + /** + * Retrieve the syllable belonging to this target. + * + * @author Marc Schröder + * + */ + public static class SyllableNavigator implements TargetElementNavigator { + public Element getElement(Target target) { + Element segment = target.getMaryxmlElement(); + if (segment == null) + return null; + if (!segment.getTagName().equals(MaryXML.PHONE)) + return null; + Element syllable = (Element) segment.getParentNode(); + if (syllable != null) { + assert syllable.getTagName().equals(MaryXML.SYLLABLE) : "Unexpected tag name: expected " + MaryXML.SYLLABLE + + ", got " + syllable.getTagName(); + } + return syllable; + } + } + + /** + * Retrieve the syllable before the syllable belonging to this target. + * + * @author Marc Schröder + * + */ + public static class PrevSyllableNavigator implements TargetElementNavigator { + public Element getElement(Target target) { + Element segment = target.getMaryxmlElement(); + if (segment == null) + return null; + Element current; + if (segment.getTagName().equals(MaryXML.PHONE)) { + Element syllable = (Element) segment.getParentNode(); + if (syllable == null) + return null; + current = syllable; + } else { // boundary + current = segment; + } + Element sentence = (Element) MaryDomUtils.getAncestor(segment, MaryXML.SENTENCE); + if (sentence == null) + return null; + TreeWalker tw = MaryDomUtils.createTreeWalker(sentence, MaryXML.SYLLABLE); + tw.setCurrentNode(current); + Element previous = (Element) tw.previousNode(); + if (previous != null) { + assert previous.getTagName().equals(MaryXML.SYLLABLE) : "Unexpected tag name: expected " + MaryXML.SYLLABLE + + ", got " + previous.getTagName(); + } + return previous; + } + } + + /** + * Retrieve the syllable two before the syllable belonging to this target. + * + * @author Marc Schröder + * + */ + public static class PrevPrevSyllableNavigator implements TargetElementNavigator { + public Element getElement(Target target) { + Element segment = target.getMaryxmlElement(); + if (segment == null) + return null; + Element current; + if (segment.getTagName().equals(MaryXML.PHONE)) { + Element syllable = (Element) segment.getParentNode(); + if (syllable == null) + return null; + current = syllable; + } else { // boundary + current = segment; + } + Element sentence = (Element) MaryDomUtils.getAncestor(segment, MaryXML.SENTENCE); + if (sentence == null) + return null; + TreeWalker tw = MaryDomUtils.createTreeWalker(sentence, MaryXML.SYLLABLE); + tw.setCurrentNode(current); + Element previous = (Element) tw.previousNode(); + Element pp = (Element) tw.previousNode(); + if (pp != null) { + assert pp.getTagName().equals(MaryXML.SYLLABLE) : "Unexpected tag name: expected " + MaryXML.SYLLABLE + ", got " + + pp.getTagName(); + } + return pp; + } + } + + /** + * Retrieve the syllable following the syllable belonging to this target. + * + * @author Marc Schröder + * + */ + public static class NextSyllableNavigator implements TargetElementNavigator { + public Element getElement(Target target) { + Element segment = target.getMaryxmlElement(); + if (segment == null) + return null; + Element current; + if (segment.getTagName().equals(MaryXML.PHONE)) { + Element syllable = (Element) segment.getParentNode(); + if (syllable == null) + return null; + current = syllable; + } else { // boundary + current = segment; + } + Element sentence = (Element) MaryDomUtils.getAncestor(segment, MaryXML.SENTENCE); + if (sentence == null) + return null; + TreeWalker tw = MaryDomUtils.createTreeWalker(sentence, MaryXML.SYLLABLE); + tw.setCurrentNode(current); + Element next = (Element) tw.nextNode(); + if (next != null) { + assert next.getTagName().equals(MaryXML.SYLLABLE) : "Unexpected tag name: expected " + MaryXML.SYLLABLE + + ", got " + next.getTagName(); + } + return next; + } + } + + /** + * Retrieve the syllable two after the syllable belonging to this target. + * + * @author Marc Schröder + * + */ + public static class NextNextSyllableNavigator implements TargetElementNavigator { + public Element getElement(Target target) { + Element segment = target.getMaryxmlElement(); + if (segment == null) + return null; + Element current; + if (segment.getTagName().equals(MaryXML.PHONE)) { + Element syllable = (Element) segment.getParentNode(); + if (syllable == null) + return null; + current = syllable; + } else { // boundary + current = segment; + } + Element sentence = (Element) MaryDomUtils.getAncestor(segment, MaryXML.SENTENCE); + if (sentence == null) + return null; + TreeWalker tw = MaryDomUtils.createTreeWalker(sentence, MaryXML.SYLLABLE); + tw.setCurrentNode(current); + Element next = (Element) tw.nextNode(); + Element nn = (Element) tw.nextNode(); + if (nn != null) { + assert nn.getTagName().equals(MaryXML.SYLLABLE) : "Unexpected tag name: expected " + MaryXML.SYLLABLE + ", got " + + nn.getTagName(); + } + return nn; + } + } + + /** + * Retrieve the word belonging to this target. + * + * @author Marc Schröder + * + */ + public static class WordNavigator implements TargetElementNavigator { + public Element getElement(Target target) { + Element segment = target.getMaryxmlElement(); + if (segment == null) + return null; + Element word = (Element) MaryDomUtils.getAncestor(segment, MaryXML.TOKEN); + if (word != null) { + assert word.getTagName().equals(MaryXML.TOKEN) : "Unexpected tag name: expected " + MaryXML.TOKEN + ", got " + + word.getTagName(); + } + return word; + } + } + + /** Last syllable in phrase. */ + public static class LastSyllableInPhraseNavigator implements TargetElementNavigator { + public Element getElement(Target target) { + Element segment = target.getMaryxmlElement(); + if (segment == null) + return null; + Element phrase = (Element) MaryDomUtils.getAncestor(segment, MaryXML.PHRASE); + if (phrase == null) + return null; + TreeWalker tw = MaryDomUtils.createTreeWalker(phrase, MaryXML.SYLLABLE); + Element last = (Element) tw.lastChild(); + if (last != null) { + assert last.getTagName().equals(MaryXML.SYLLABLE) : "Unexpected tag name: expected " + MaryXML.SYLLABLE + + ", got " + last.getTagName(); + } + return last; + } + } + + public static class NextWordNavigator implements TargetElementNavigator { + public Element getElement(Target target) { + Element segment = target.getMaryxmlElement(); + if (segment == null) + return null; + Element current; + if (segment.getTagName().equals(MaryXML.PHONE)) { + Element word = (Element) MaryDomUtils.getAncestor(segment, MaryXML.TOKEN); + if (word == null) + return null; + current = word; + } else { // boundary + current = segment; + } + Element sentence = (Element) MaryDomUtils.getAncestor(segment, MaryXML.SENTENCE); + if (sentence == null) + return null; + TreeWalker tw = MaryDomUtils.createTreeWalker(sentence, MaryXML.TOKEN); + tw.setCurrentNode(current); + // The next word is the next token with a "ph" attribute: + Element nextWord = null; + Element nextToken; + while ((nextToken = (Element) tw.nextNode()) != null) { + if (nextToken.hasAttribute("ph")) { + nextWord = nextToken; + break; + } + } + if (nextWord != null) { + assert nextWord.getTagName().equals(MaryXML.TOKEN) : "Unexpected tag name: expected " + MaryXML.TOKEN + ", got " + + nextWord.getTagName(); + } + return nextWord; + } + } + + public static class PrevWordNavigator implements TargetElementNavigator { + public Element getElement(Target target) { + Element segment = target.getMaryxmlElement(); + if (segment == null) + return null; + Element current; + if (segment.getTagName().equals(MaryXML.PHONE)) { + Element word = (Element) MaryDomUtils.getAncestor(segment, MaryXML.TOKEN); + if (word == null) + return null; + current = word; + } else { // boundary + current = segment; + } + Element sentence = (Element) MaryDomUtils.getAncestor(segment, MaryXML.SENTENCE); + if (sentence == null) + return null; + TreeWalker tw = MaryDomUtils.createTreeWalker(sentence, MaryXML.TOKEN); + tw.setCurrentNode(current); + // The next word is the next token with a "ph" attribute: + Element prevWord = null; + Element prevToken; + while ((prevToken = (Element) tw.previousNode()) != null) { + if (prevToken.hasAttribute("ph")) { + prevWord = prevToken; + break; + } + } + if (prevWord != null) { + assert prevWord.getTagName().equals(MaryXML.TOKEN) : "Unexpected tag name: expected " + MaryXML.TOKEN + ", got " + + prevWord.getTagName(); + } + return prevWord; + } + } + + public static class FirstSegmentNextWordNavigator implements TargetElementNavigator { + public Element getElement(Target target) { + Element segment = target.getMaryxmlElement(); + if (segment == null) + return null; + Element current; + if (segment.getTagName().equals(MaryXML.PHONE)) { + Element word = (Element) MaryDomUtils.getAncestor(segment, MaryXML.TOKEN); + if (word == null) + return null; + current = word; + } else { // boundary + current = segment; + } + Element sentence = (Element) MaryDomUtils.getAncestor(segment, MaryXML.SENTENCE); + if (sentence == null) + return null; + TreeWalker tw = MaryDomUtils.createTreeWalker(sentence, MaryXML.TOKEN); + tw.setCurrentNode(current); + // The next word is the next token with a "ph" attribute: + Element nextWord = null; + Element nextToken; + while ((nextToken = (Element) tw.nextNode()) != null) { + if (nextToken.hasAttribute("ph")) { + nextWord = nextToken; + break; + } + } + if (nextWord == null) { + return null; + } + assert nextWord.getTagName().equals(MaryXML.TOKEN) : "Unexpected tag name: expected " + MaryXML.TOKEN + ", got " + + nextWord.getTagName(); + TreeWalker sw = MaryDomUtils.createTreeWalker(nextWord, MaryXML.PHONE); + Element first = (Element) sw.firstChild(); + if (first != null) { + assert first.getTagName().equals(MaryXML.PHONE) : "Unexpected tag name: expected " + MaryXML.PHONE + ", got " + + first.getTagName(); + } + return first; + } + } + + public static class LastWordInSentenceNavigator implements TargetElementNavigator { + public Element getElement(Target target) { + Element segment = target.getMaryxmlElement(); + if (segment == null) + return null; + Element sentence = (Element) MaryDomUtils.getAncestor(segment, MaryXML.SENTENCE); + if (sentence == null) + return null; + TreeWalker tw = MaryDomUtils.createTreeWalker(sentence, MaryXML.TOKEN); + Element lastWord = null; + Element lastToken = (Element) tw.lastChild(); + // The last word is the lastToken which has a "ph" attribute: + while (lastToken != null) { + if (lastToken.hasAttribute("ph")) { + lastWord = lastToken; + break; + } + lastToken = (Element) tw.previousNode(); + } + + if (lastWord != null) { + assert lastWord.getTagName().equals(MaryXML.TOKEN) : "Unexpected tag name: expected " + MaryXML.TOKEN + ", got " + + lastWord.getTagName(); + } + return lastWord; + } + } + + // no instances + protected MaryGenericFeatureProcessors() { + } + + /** + * flite never returns an int more than 19 from a feature processor, we duplicate that behavior in the processors so that our + * tests will match. let's keep this number as a constant for better overview + */ + private static final int RAIL_LIMIT = 19; + + private static final String[] ZERO_TO_NINETEEN = new String[] { "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", + "12", "13", "14", "15", "16", "17", "18", "19" }; + + /** + * Indicate whether a unit is an edge unit, which is never the case for a target. + */ + public static class Edge implements ByteValuedFeatureProcessor { + public String getName() { + return "edge"; + } + + public String[] getValues() { + return new String[] { "0", "start", "end" }; + } + + /** + * This processor always returns 0 for targets. + */ + public byte process(Target target) { + return (byte) 0; + } + + } + + /** + * Is the given half phone target a left or a right half? + * + * @author Marc Schröder + * + */ + public static class HalfPhoneLeftRight implements ByteValuedFeatureProcessor { + protected ByteStringTranslator values; + + /** + * Initialise a HalfPhoneLeftRight feature processor. + */ + public HalfPhoneLeftRight() { + this.values = new ByteStringTranslator(new String[] { "0", "L", "R" }); + } + + public String getName() { + return "halfphone_lr"; + } + + public String[] getValues() { + return values.getStringValues(); + } + + public byte process(Target target) { + if (!(target instanceof HalfPhoneTarget)) + return 0; + HalfPhoneTarget hpTarget = (HalfPhoneTarget) target; + String value = (hpTarget.isLeftHalf() ? "L" : "R"); + return values.get(value); + } + } + + /** + * Sentence Style for the given target + * + * @author Sathish Chandra Pammi + * + */ + + public static class Style implements ByteValuedFeatureProcessor { + protected ByteStringTranslator values; + protected TargetElementNavigator navigator; + protected final String styleTagName = "style"; + + /** + * Initialize a speaking Style feature processor. + */ + public Style() { + this.values = new ByteStringTranslator(new String[] { "0", "neutral", "poker", "happy", "sad", "angry", "excited" }); + this.navigator = new SegmentNavigator(); + } + + public String getName() { + return styleTagName; + } + + public String[] getValues() { + return values.getStringValues(); + } + + public byte process(Target target) { + String style = null; + Element segment = target.getMaryxmlElement(); + if (segment != null) { + Element prosody = (Element) MaryDomUtils.getClosestAncestorWithAttribute(segment, MaryXML.PROSODY, styleTagName); + if (prosody != null) { + style = prosody.getAttribute(styleTagName); + } + } + if (style == null || style.equals("")) + style = "0"; + if (values.contains(style)) { + return values.get(style); + } else { // silently ignore unknown values + return 0; + } + } + } + + /** + * Checks to see if the given syllable is accented. + */ + public static class Accented implements ByteValuedFeatureProcessor { + protected String name; + protected TargetElementNavigator navigator; + + public Accented(String name, TargetElementNavigator syllableNavigator) { + this.name = name; + this.navigator = syllableNavigator; + } + + public String getName() { + return name; + } + + public String[] getValues() { + return new String[] { "0", "1" }; + } + + /** + * Performs some processing on the given item. + * + * @param target + * the target to process + * @return "1" if the syllable is accented; otherwise "0" + */ + public byte process(Target target) { + Element syllable = navigator.getElement(target); + if (syllable != null && syllable.hasAttribute("accent")) { + return (byte) 1; + } else { + return (byte) 0; + } + } + } + + /** + * Checks to see if the given syllable is stressed. + */ + public static class Stressed implements ByteValuedFeatureProcessor { + protected String name; + protected TargetElementNavigator navigator; + + public Stressed(String name, TargetElementNavigator syllableNavigator) { + this.name = name; + this.navigator = syllableNavigator; + } + + public String getName() { + return name; + } + + public String[] getValues() { + return new String[] { "0", "1" }; + } + + /** + * Performs some processing on the given item. + * + * @param target + * the target to process + * @return "1" if the syllable is stressed; otherwise "0" + */ + public byte process(Target target) { + Element syllable = navigator.getElement(target); + if (syllable == null) + return 0; + String value = syllable.getAttribute("stress"); + if (value.equals("")) + return 0; + byte stressValue = Byte.parseByte(value); + if (stressValue > 1) { + // out of range, set to 1 + stressValue = 1; + } + return stressValue; + } + } + + /** + * Syllable tone for the given target + * + * @author sathish pammi + * + */ + public static class SyllableTone implements ByteValuedFeatureProcessor { + protected String name; + protected TargetElementNavigator navigator; + + public SyllableTone(String name, TargetElementNavigator syllableNavigator) { + this.name = name; + this.navigator = syllableNavigator; + } + + public String getName() { + return name; + } + + public String[] getValues() { + return new String[] { "0", "1", "2", "3", "4" }; + } + + /** + * Performs some processing on the given item. + * + * @param target + * the target to process + * @return tone value + */ + public byte process(Target target) { + Element syllable = navigator.getElement(target); + if (syllable == null) + return 0; + String value = syllable.getAttribute("tone"); + if (value.equals("")) + return 0; + byte toneValue = Byte.parseByte(value); + if (toneValue > 4 || toneValue < 1) { + // out of range, set to 0 + toneValue = 0; + } + return toneValue; + } + } + + /** + * Returns as a byte the number of phrases in the current sentence. + */ + public static class SentenceNumPhrases implements ByteValuedFeatureProcessor { + public SentenceNumPhrases() { + } + + public String getName() { + return "sentence_numphrases"; + } + + public String[] getValues() { + return ZERO_TO_NINETEEN; + } + + /** + * + * @return the number of phrases in the sentence + */ + public byte process(Target target) { + Element segment = target.getMaryxmlElement(); + if (segment == null) + return (byte) 0; + Element sentence = (Element) MaryDomUtils.getAncestor(segment, MaryXML.SENTENCE); + if (sentence == null) + return (byte) 0; + TreeWalker tw = MaryDomUtils.createTreeWalker(sentence, MaryXML.PHRASE); + int count = 0; + Element e; + while ((e = (Element) tw.nextNode()) != null && count < RAIL_LIMIT) { + count++; + } + return (byte) count; + } + } + + /** + * Returns as an Integer the number of words in the current sentence. This is a feature processor. A feature processor takes + * an item, performs some sort of processing on the item and returns an object. + */ + public static class SentenceNumWords implements ByteValuedFeatureProcessor { + public SentenceNumWords() { + } + + public String getName() { + return "sentence_numwords"; + } + + public String[] getValues() { + return ZERO_TO_NINETEEN; + } + + /** + * @return the number of words in the sentence + */ + public byte process(Target target) { + Element segment = target.getMaryxmlElement(); + if (segment == null) + return (byte) 0; + Element sentence = (Element) MaryDomUtils.getAncestor(segment, MaryXML.SENTENCE); + if (sentence == null) + return (byte) 0; + TreeWalker tw = MaryDomUtils.createTreeWalker(sentence, MaryXML.TOKEN); + int count = 0; + Element e; + while ((e = (Element) tw.nextNode()) != null && count < RAIL_LIMIT) { + // only tokens with a "ph" attribute count as words: + if (e.hasAttribute("ph")) + count++; + } + return (byte) count; + } + } + + /** + * Returns as a byte the number of phrases in the current sentence. + */ + public static class PhraseNumSyls implements ByteValuedFeatureProcessor { + public PhraseNumSyls() { + } + + public String getName() { + return "phrase_numsyls"; + } + + public String[] getValues() { + return ZERO_TO_NINETEEN; + } + + /** + * @return the number of words in the phrase + */ + public byte process(Target target) { + Element segment = target.getMaryxmlElement(); + if (segment == null) + return (byte) 0; + Element phrase = (Element) MaryDomUtils.getAncestor(segment, MaryXML.PHRASE); + if (phrase == null) + return (byte) 0; + TreeWalker tw = MaryDomUtils.createTreeWalker(phrase, MaryXML.SYLLABLE); + int count = 0; + Element e; + while ((e = (Element) tw.nextNode()) != null && count < RAIL_LIMIT) { + count++; + } + return (byte) count; + } + } + + /** + * Returns as a byte the number of words in the current phrase. + */ + public static class PhraseNumWords implements ByteValuedFeatureProcessor { + public PhraseNumWords() { + } + + public String getName() { + return "phrase_numwords"; + } + + public String[] getValues() { + return ZERO_TO_NINETEEN; + } + + /** + * Performs some processing on the given item. + * + * @param item + * the item to process + * + * @return the number of words in the phrase + */ + public byte process(Target target) { + Element segment = target.getMaryxmlElement(); + if (segment == null) + return (byte) 0; + Element phrase = (Element) MaryDomUtils.getAncestor(segment, MaryXML.PHRASE); + if (phrase == null) + return (byte) 0; + TreeWalker tw = MaryDomUtils.createTreeWalker(phrase, MaryXML.TOKEN); + int count = 0; + Element e; + while ((e = (Element) tw.nextNode()) != null && count < RAIL_LIMIT) { + count++; + } + return (byte) count; + } + } + + /** + * Returns as an Integer the number of syllables in the given word. This is a feature processor. A feature processor takes an + * item, performs some sort of processing on the item and returns an object. + */ + public static class WordNumSyls implements ByteValuedFeatureProcessor { + public WordNumSyls() { + } + + public String getName() { + return "word_numsyls"; + } + + public String[] getValues() { + return ZERO_TO_NINETEEN; + } + + /** + * @return the number of syllables in the given word + */ + public byte process(Target target) { + Element segment = target.getMaryxmlElement(); + if (segment == null) + return (byte) 0; + Element word = (Element) MaryDomUtils.getAncestor(segment, MaryXML.TOKEN); + if (word == null) + return (byte) 0; + TreeWalker tw = MaryDomUtils.createTreeWalker(word, MaryXML.SYLLABLE); + int count = 0; + Element e; + while ((e = (Element) tw.nextNode()) != null && count < RAIL_LIMIT) { + count++; + } + return (byte) count; + } + } + + /** + * Returns as a byte the number of segments in the given word. + */ + public static class WordNumSegs implements ByteValuedFeatureProcessor { + public WordNumSegs() { + } + + public String getName() { + return "word_numsegs"; + } + + public String[] getValues() { + return ZERO_TO_NINETEEN; + } + + /** + * @return the number of segments in the given word + */ + public byte process(Target target) { + Element segment = target.getMaryxmlElement(); + if (segment == null) + return (byte) 0; + Element word = (Element) MaryDomUtils.getAncestor(segment, MaryXML.TOKEN); + if (word == null) + return (byte) 0; + TreeWalker tw = MaryDomUtils.createTreeWalker(word, MaryXML.PHONE); + int count = 0; + Element e; + while ((e = (Element) tw.nextNode()) != null && count < RAIL_LIMIT) { + count++; + } + return (byte) count; + } + } + + /** + * Returns as an Integer the number of segments in the current syllable. This is a feature processor. A feature processor + * takes an item, performs some sort of processing on the item and returns an object. + */ + public static class SylNumSegs implements ByteValuedFeatureProcessor { + public SylNumSegs() { + } + + public String getName() { + return "syl_numsegs"; + } + + public String[] getValues() { + return ZERO_TO_NINETEEN; + } + + /** + * Performs some processing on the given item. + * + * @param item + * the item to process + * + * @return the number of segments in the given word + */ + public byte process(Target target) { + Element segment = target.getMaryxmlElement(); + if (segment == null) + return (byte) 0; + if (!segment.getTagName().equals(MaryXML.PHONE)) + return 0; + Element syllable = (Element) segment.getParentNode(); + if (syllable == null) + return (byte) 0; + TreeWalker tw = MaryDomUtils.createTreeWalker(syllable, MaryXML.PHONE); + int count = 0; + Element e; + while ((e = (Element) tw.nextNode()) != null && count < RAIL_LIMIT) { + count++; + } + return (byte) count; + } + } + + /** + * @deprecated, use SegsFromSylStart instead + */ + public static class PosInSyl extends SegsFromSylStart { + public PosInSyl() { + super(); + } + + public String getName() { + return "pos_in_syl"; + } + } + + /** + * Finds the position of the phone in the syllable. + */ + public static class SegsFromSylStart implements ByteValuedFeatureProcessor { + public SegsFromSylStart() { + } + + public String getName() { + return "segs_from_syl_start"; + } + + public String[] getValues() { + return ZERO_TO_NINETEEN; + } + + /** + * @param target + * the target to process + * @return the position of the phone in the syllable + */ + public byte process(Target target) { + Element segment = target.getMaryxmlElement(); + if (segment == null) + return (byte) 0; + if (!segment.getTagName().equals(MaryXML.PHONE)) + return 0; + int count = 0; + Element e = segment; + while ((e = MaryDomUtils.getPreviousSiblingElement(e)) != null && count < RAIL_LIMIT) { + count++; + } + return (byte) count; + } + } + + /** + * Finds the position of the phone from the end of the syllable. + */ + public static class SegsFromSylEnd implements ByteValuedFeatureProcessor { + public SegsFromSylEnd() { + } + + public String getName() { + return "segs_from_syl_end"; + } + + public String[] getValues() { + return ZERO_TO_NINETEEN; + } + + /** + * @param target + * the target to process + * @return the position of the phone in the syllable + */ + public byte process(Target target) { + Element segment = target.getMaryxmlElement(); + if (segment == null) + return (byte) 0; + if (!segment.getTagName().equals(MaryXML.PHONE)) + return 0; + int count = 0; + Element e = segment; + while ((e = MaryDomUtils.getNextSiblingElement(e)) != null && count < RAIL_LIMIT) { + count++; + } + return (byte) count; + } + } + + /** + * Finds the position of the segment from the start of the word. + */ + public static class SegsFromWordStart implements ByteValuedFeatureProcessor { + public SegsFromWordStart() { + } + + public String getName() { + return "segs_from_word_start"; + } + + public String[] getValues() { + return ZERO_TO_NINETEEN; + } + + /** + * @param target + * the target to process + * @return the position of the phone in the syllable + */ + public byte process(Target target) { + Element segment = target.getMaryxmlElement(); + if (segment == null) + return (byte) 0; + Element word = (Element) MaryDomUtils.getAncestor(segment, MaryXML.TOKEN); + if (word == null) + return (byte) 0; + TreeWalker tw = MaryDomUtils.createTreeWalker(word, MaryXML.PHONE); + tw.setCurrentNode(segment); + int count = 0; + Element e; + while ((e = (Element) tw.previousNode()) != null && count < RAIL_LIMIT) { + count++; + } + return (byte) count; + } + } + + /** + * Finds the position of the segment from the end of the word. + */ + public static class SegsFromWordEnd implements ByteValuedFeatureProcessor { + public SegsFromWordEnd() { + } + + public String getName() { + return "segs_from_word_end"; + } + + public String[] getValues() { + return ZERO_TO_NINETEEN; + } + + /** + * Performs some processing on the given item. + * + * @param target + * the target to process + * @return the position of the phone in the syllable + */ + public byte process(Target target) { + Element segment = target.getMaryxmlElement(); + if (segment == null) + return (byte) 0; + Element word = (Element) MaryDomUtils.getAncestor(segment, MaryXML.TOKEN); + if (word == null) + return (byte) 0; + TreeWalker tw = MaryDomUtils.createTreeWalker(word, MaryXML.PHONE); + tw.setCurrentNode(segment); + int count = 0; + Element e; + while ((e = (Element) tw.nextNode()) != null && count < RAIL_LIMIT) { + count++; + } + return (byte) count; + } + } + + /** + * Finds the position of the syllable from the start of the word. + */ + public static class SylsFromWordStart implements ByteValuedFeatureProcessor { + public SylsFromWordStart() { + } + + public String getName() { + return "syls_from_word_start"; + } + + public String[] getValues() { + return ZERO_TO_NINETEEN; + } + + /** + * @param target + * the target to process + * @return the position of the syllable in the word + */ + public byte process(Target target) { + Element segment = target.getMaryxmlElement(); + if (segment == null) + return (byte) 0; + if (!segment.getTagName().equals(MaryXML.PHONE)) + return 0; + Element syllable = (Element) segment.getParentNode(); + if (syllable == null) + return (byte) 0; + int count = 0; + Element e = syllable; + while ((e = MaryDomUtils.getPreviousSiblingElement(e)) != null && count < RAIL_LIMIT) { + count++; + } + return (byte) count; + } + } + + /** + * Finds the position of the syllable from the end of the word. + */ + public static class SylsFromWordEnd implements ByteValuedFeatureProcessor { + public SylsFromWordEnd() { + } + + public String getName() { + return "syls_from_word_end"; + } + + public String[] getValues() { + return ZERO_TO_NINETEEN; + } + + /** + * @param target + * the target to process + * @return the position of the phone in the syllable + */ + public byte process(Target target) { + Element segment = target.getMaryxmlElement(); + if (segment == null) + return (byte) 0; + if (!segment.getTagName().equals(MaryXML.PHONE)) + return 0; + Element syllable = (Element) segment.getParentNode(); + if (syllable == null) + return (byte) 0; + int count = 0; + Element e = syllable; + while ((e = MaryDomUtils.getNextSiblingElement(e)) != null && count < RAIL_LIMIT) { + count++; + } + return (byte) count; + } + } + + /** + * Determines the break level after this syllable. + */ + public static class SylBreak implements ByteValuedFeatureProcessor { + protected String name; + protected TargetElementNavigator navigator; + + public SylBreak(String name, TargetElementNavigator syllableNavigator) { + this.name = name; + this.navigator = syllableNavigator; + } + + public String getName() { + return name; + } + + /** + * "4" for a big break, "3" for a break; "1" = word-final; "0" = within-word + */ + public String[] getValues() { + return new String[] { "0", "1", "unused", "3", "4" }; + } + + /** + * @param target + * the target to process + * @return the break level after the syllable returned by syllableNavigator + */ + public byte process(Target target) { + Element syllable = navigator.getElement(target); + if (syllable == null) + return 0; + // is there another syllable following in the token? + if (MaryDomUtils.getNextSiblingElement(syllable) != null) + return 0; + // else, it is at least word-final. + Element word = (Element) syllable.getParentNode(); + if (word == null) + return 0; + assert word.getTagName().equals(MaryXML.TOKEN) : "Unexpected tag name: expected " + MaryXML.TOKEN + ", got " + + word.getTagName(); + Element sentence = (Element) MaryDomUtils.getAncestor(word, MaryXML.SENTENCE); + if (sentence == null) + return 0; + TreeWalker tw = MaryDomUtils.createTreeWalker(sentence, MaryXML.TOKEN, MaryXML.BOUNDARY); + tw.setCurrentNode(word); + // The next word is the next token with a "ph" attribute: + Element e; + while ((e = (Element) tw.nextNode()) != null) { + if (e.getTagName().equals(MaryXML.BOUNDARY) || e.getTagName().equals(MaryXML.TOKEN) && e.hasAttribute("ph")) + break; + } + if (e == null) { + // we are the last token in the sentence, but there is no boundary... + // OK, let's say it is sentence-final anyway: + return 4; + } + if (e.getTagName().equals(MaryXML.TOKEN)) { + // a word follows + return 1; + } + assert e.getTagName().equals(MaryXML.BOUNDARY) : "Unexpected tag name: expected " + MaryXML.BOUNDARY + ", got " + + e.getTagName(); + String bi = e.getAttribute("breakindex"); + if (bi.equals("")) { + // no breakindex + return 1; + } + try { + int ibi = Integer.parseInt(bi); + if (ibi >= 4) + return 4; + if (ibi == 3) + return 3; + } catch (NumberFormatException nfe) { + } + // default: a word boundary + return 1; + } + } + + /** + * Classifies the the syllable as single, initial, mid or final. + */ + public static class PositionType implements ByteValuedFeatureProcessor { + protected TargetElementNavigator navigator; + protected ByteStringTranslator values; + + public PositionType() { + values = new ByteStringTranslator(new String[] { "0", "single", "final", "initial", "mid" }); + navigator = new SyllableNavigator(); + } + + public String getName() { + return "position_type"; + } + + public String[] getValues() { + return values.getStringValues(); + } + + /** + * @param target + * the target to process + * @return classifies the syllable as "single", "final", "initial" or "mid" + */ + public byte process(Target target) { + Element syllable = navigator.getElement(target); + if (syllable == null) + return 0; + String type; + if (MaryDomUtils.getNextSiblingElement(syllable) == null) { + if (MaryDomUtils.getPreviousSiblingElement(syllable) == null) { + type = "single"; + } else { + type = "final"; + } + } else if (MaryDomUtils.getPreviousSiblingElement(syllable) == null) { + type = "initial"; + } else { + type = "mid"; + } + return values.get(type); + } + } + + /** + * Checks if segment is a pause. + */ + public static class IsPause implements ByteValuedFeatureProcessor { + protected TargetElementNavigator navigator; + protected String name; + + public IsPause(String name, TargetElementNavigator segmentNavigator) { + this.name = name; + this.navigator = segmentNavigator; + } + + public String getName() { + return name; + } + + public String[] getValues() { + return new String[] { "0", "1" }; + } + + /** + * Check if segment is a pause + * + * @param target + * the target to process + * @return 0 if false, 1 if true + */ + public byte process(Target target) { + Element seg = navigator.getElement(target); + if (seg == null) + return 0; + if (seg.getTagName().equals(MaryXML.BOUNDARY)) + return 1; + return 0; + } + } + + public static class BreakIndex implements ByteValuedFeatureProcessor { + protected ByteStringTranslator values; + + public BreakIndex() { + values = new ByteStringTranslator(new String[] { "0", "1", "2", "3", "4", "5", "6" }); + } + + public String getName() { + return "breakindex"; + } + + public String[] getValues() { + return values.getStringValues(); + } + + public byte process(Target target) { + Element segment = target.getMaryxmlElement(); + if (segment == null) + return 0; + Element word = (Element) MaryDomUtils.getAncestor(segment, MaryXML.TOKEN); + if (word == null) + return 0; + // is there another segment following in the token? + TreeWalker tww = MaryDomUtils.createTreeWalker(word, MaryXML.PHONE); + tww.setCurrentNode(segment); + if (tww.nextNode() != null) + return 0; + // else, it is at least word-final. + Element sentence = (Element) MaryDomUtils.getAncestor(word, MaryXML.SENTENCE); + if (sentence == null) + return 0; + TreeWalker tw = MaryDomUtils.createTreeWalker(sentence, MaryXML.TOKEN, MaryXML.BOUNDARY); + tw.setCurrentNode(word); + // The next word is the next token with a "ph" attribute: + Element e; + while ((e = (Element) tw.nextNode()) != null) { + if (e.getTagName().equals(MaryXML.BOUNDARY) || e.getTagName().equals(MaryXML.TOKEN) && e.hasAttribute("ph")) + break; + } + if (e == null) { + // we are the last token in the sentence, but there is no boundary... + // OK, let's say it is sentence-final anyway: + return 4; + } + if (e.getTagName().equals(MaryXML.TOKEN)) { + // a word follows + return 1; + } + assert e.getTagName().equals(MaryXML.BOUNDARY) : "Unexpected tag name: expected " + MaryXML.BOUNDARY + ", got " + + e.getTagName(); + String bi = e.getAttribute("breakindex"); + if (bi.equals("")) { + // no breakindex + return 1; + } + try { + int ibi = Integer.parseInt(bi); + if (ibi > 6) + ibi = 6; + if (ibi < 2) + ibi = 2; + return (byte) ibi; + } catch (NumberFormatException nfe) { + } + // default: a word boundary + return 1; + } + } + + /** + * The ToBI accent of the current syllable. + */ + public static class TobiAccent implements ByteValuedFeatureProcessor { + protected String name; + protected TargetElementNavigator navigator; + protected ByteStringTranslator values; + + public TobiAccent(String name, TargetElementNavigator syllableNavigator) { + this.name = name; + this.navigator = syllableNavigator; + this.values = new ByteStringTranslator(new String[] { "0", "*", "H*", "!H*", "^H*", "L*", "L+H*", "L*+H", "L+!H*", + "L*+!H", "L+^H*", "L*+^H", "H+L*", "H+!H*", "H+^H*", "!H+!H*", "^H+!H*", "^H+^H*", "H*+L", "!H*+L" }); + } + + public String getName() { + return name; + } + + public String[] getValues() { + return values.getStringValues(); + } + + /** + * For the given syllable item, return its tobi accent, or 0 if there is none. + */ + public byte process(Target target) { + Element syllable = navigator.getElement(target); + if (syllable == null) + return 0; + String accent = syllable.getAttribute("accent"); + if (accent.equals("")) { + return 0; + } + return values.get(accent); + } + } + + /** + * The ToBI endtone associated with the current syllable. + */ + public static class TobiEndtone implements ByteValuedFeatureProcessor { + protected String name; + protected TargetElementNavigator navigator; + protected ByteStringTranslator values; + + public TobiEndtone(String name, TargetElementNavigator syllableNavigator) { + this.name = name; + this.navigator = syllableNavigator; + this.values = new ByteStringTranslator(new String[] { "0", "H-", "!H-", "L-", "H-%", "!H-%", "H-^H%", "!H-^H%", + "L-H%", "L-%", "L-L%", "H-H%", "H-L%" }); + } + + public String getName() { + return name; + } + + public String[] getValues() { + return values.getStringValues(); + } + + /** + * For the given syllable item, return its tobi end tone, or 0 if there is none. + */ + public byte process(Target target) { + Element syllable = navigator.getElement(target); + if (syllable == null) + return 0; + Element sentence = (Element) MaryDomUtils.getAncestor(syllable, MaryXML.SENTENCE); + if (sentence == null) + return 0; + TreeWalker tw = MaryDomUtils.createTreeWalker(sentence, MaryXML.SYLLABLE, MaryXML.BOUNDARY); + tw.setCurrentNode(syllable); + Element e = (Element) tw.nextNode(); + if (e == null) + return 0; + if (e.getTagName().equals(MaryXML.SYLLABLE)) + return 0; + assert e.getTagName().equals(MaryXML.BOUNDARY) : "Unexpected tag name: expected " + MaryXML.BOUNDARY + ", got " + + e.getTagName(); + String endtone = e.getAttribute("tone"); + if (endtone.equals("")) { + return 0; + } + return values.get(endtone); + } + } + + /** + * The next ToBI accent following the current syllable in the current phrase. + */ + public static class NextAccent extends TobiAccent { + public NextAccent() { + super("next_accent", null); + } + + /** + * Search for an accented syllable, and return its tobi accent, or 0 if there is none. + */ + public byte process(Target target) { + Element segment = target.getMaryxmlElement(); + if (segment == null) + return 0; + Element current; + if (segment.getTagName().equals(MaryXML.PHONE)) { + Element syllable = (Element) segment.getParentNode(); + if (syllable == null) + return 0; + current = syllable; + } else { // boundary + current = segment; + } + Element phrase = (Element) MaryDomUtils.getAncestor(current, MaryXML.PHRASE); + if (phrase == null) + return 0; + TreeWalker tw = MaryDomUtils.createTreeWalker(phrase, MaryXML.SYLLABLE); + tw.setCurrentNode(current); + Element s; + while ((s = (Element) tw.nextNode()) != null) { + if (s.hasAttribute("accent")) { + String accent = s.getAttribute("accent"); + return values.get(accent); + } + } + return 0; + } + } + + /** + * The previous ToBI accent preceding the current syllable in the current phrase. + */ + public static class PrevAccent extends TobiAccent { + public PrevAccent() { + super("prev_accent", null); + } + + /** + * Search for an accented syllable, and return its tobi accent, or 0 if there is none. + */ + public byte process(Target target) { + Element segment = target.getMaryxmlElement(); + if (segment == null) + return 0; + Element current; + if (segment.getTagName().equals(MaryXML.PHONE)) { + Element syllable = (Element) segment.getParentNode(); + if (syllable == null) + return 0; + current = syllable; + } else { // boundary + current = segment; + } + Element phrase = (Element) MaryDomUtils.getAncestor(current, MaryXML.PHRASE); + if (phrase == null) + return 0; + TreeWalker tw = MaryDomUtils.createTreeWalker(phrase, MaryXML.SYLLABLE); + tw.setCurrentNode(current); + Element s; + while ((s = (Element) tw.previousNode()) != null) { + if (s.hasAttribute("accent")) { + String accent = s.getAttribute("accent"); + return values.get(accent); + } + } + return 0; + } + } + + /** + * The ToBI endtone associated with the last syllable of the current phrase. + */ + public static class PhraseEndtone extends TobiEndtone { + public PhraseEndtone() { + super("phrase_endtone", new LastSyllableInPhraseNavigator()); + } + } + + /** + * The ToBI endtone associated with the last syllable of the previous phrase. + */ + public static class PrevPhraseEndtone extends TobiEndtone { + public PrevPhraseEndtone() { + super("prev_phrase_endtone", null); + } + + public byte process(Target target) { + Element segment = target.getMaryxmlElement(); + if (segment == null) + return 0; + Element phrase = (Element) MaryDomUtils.getAncestor(segment, MaryXML.PHRASE); + if (phrase == null) + return 0; + Document doc = phrase.getOwnerDocument(); + TreeWalker tw = MaryDomUtils.createTreeWalker(doc, doc, MaryXML.BOUNDARY); + tw.setCurrentNode(phrase); + Element boundary = (Element) tw.previousNode(); + if (boundary == null) + return 0; + String endtone = boundary.getAttribute("tone"); + if (endtone.equals("")) { + return 0; + } + return values.get(endtone); + } + } + + /** + * Counts the number of syllables since the start of the phrase. + */ + public static class SylsFromPhraseStart implements ByteValuedFeatureProcessor { + public SylsFromPhraseStart() { + } + + public String getName() { + return "syls_from_phrase_start"; + } + + public String[] getValues() { + return ZERO_TO_NINETEEN; + } + + /** + * @param target + * the target to process + * @return the number of syllables since the last major break + */ + public byte process(Target target) { + Element segment = target.getMaryxmlElement(); + if (segment == null) + return 0; + Element phrase = (Element) MaryDomUtils.getAncestor(segment, MaryXML.PHRASE); + if (phrase == null) + return 0; + int count = 0; + TreeWalker tw = MaryDomUtils.createTreeWalker(phrase, MaryXML.SYLLABLE); + Element syllable = (Element) MaryDomUtils.getAncestor(segment, MaryXML.SYLLABLE); + if (syllable != null) { + tw.setCurrentNode(syllable); + } else { + tw.setCurrentNode(segment); + } + Element e; + while ((e = (Element) tw.previousNode()) != null && count < RAIL_LIMIT) { + count++; + } + return (byte) count; + } + } + + /** + * Counts the number of syllables until the end of the phrase. + */ + public static class SylsFromPhraseEnd implements ByteValuedFeatureProcessor { + public SylsFromPhraseEnd() { + } + + public String getName() { + return "syls_from_phrase_end"; + } + + public String[] getValues() { + return ZERO_TO_NINETEEN; + } + + /** + * @param target + * the target to process + * @return the number of accented syllables since the last major break + */ + public byte process(Target target) { + Element segment = target.getMaryxmlElement(); + if (segment == null) + return 0; + Element phrase = (Element) MaryDomUtils.getAncestor(segment, MaryXML.PHRASE); + if (phrase == null) + return 0; + int count = 0; + TreeWalker tw = MaryDomUtils.createTreeWalker(phrase, MaryXML.SYLLABLE); + tw.setCurrentNode(segment); + Element e; + while ((e = (Element) tw.nextNode()) != null && count < RAIL_LIMIT) { + count++; + } + return (byte) count; + } + } + + /** + * Counts the number of stressed syllables since the start of the phrase. + */ + public static class StressedSylsFromPhraseStart implements ByteValuedFeatureProcessor { + public StressedSylsFromPhraseStart() { + } + + public String getName() { + return "stressed_syls_from_phrase_start"; + } + + public String[] getValues() { + return ZERO_TO_NINETEEN; + } + + /** + * @param target + * the target to process + * @return the number of stressed syllables since the last major break + */ + public byte process(Target target) { + Element segment = target.getMaryxmlElement(); + if (segment == null) + return 0; + Element phrase = (Element) MaryDomUtils.getAncestor(segment, MaryXML.PHRASE); + if (phrase == null) + return 0; + int count = 0; + TreeWalker tw = MaryDomUtils.createTreeWalker(phrase, MaryXML.SYLLABLE); + Element syllable = (Element) MaryDomUtils.getAncestor(segment, MaryXML.SYLLABLE); + if (syllable != null) { + tw.setCurrentNode(syllable); + } else { + tw.setCurrentNode(segment); + } + Element e; + while ((e = (Element) tw.previousNode()) != null && count < RAIL_LIMIT) { + String stress = e.getAttribute("stress"); + if (stress.equals("1")) + count++; + } + return (byte) count; + } + } + + /** + * Counts the number of stressed syllables until the end of the phrase. + */ + public static class StressedSylsFromPhraseEnd implements ByteValuedFeatureProcessor { + public StressedSylsFromPhraseEnd() { + } + + public String getName() { + return "stressed_syls_from_phrase_end"; + } + + public String[] getValues() { + return ZERO_TO_NINETEEN; + } + + /** + * @param target + * the target to process + * @return the number of stressed syllables since the last major break + */ + public byte process(Target target) { + Element segment = target.getMaryxmlElement(); + if (segment == null) + return 0; + Element phrase = (Element) MaryDomUtils.getAncestor(segment, MaryXML.PHRASE); + if (phrase == null) + return 0; + int count = 0; + TreeWalker tw = MaryDomUtils.createTreeWalker(phrase, MaryXML.SYLLABLE); + tw.setCurrentNode(segment); + Element e; + while ((e = (Element) tw.nextNode()) != null && count < RAIL_LIMIT) { + String stress = e.getAttribute("stress"); + if (stress.equals("1")) + count++; + } + return (byte) count; + } + } + + /** + * Counts the number of accented syllables since the start of the phrase. + */ + public static class AccentedSylsFromPhraseStart implements ByteValuedFeatureProcessor { + public AccentedSylsFromPhraseStart() { + } + + public String getName() { + return "accented_syls_from_phrase_start"; + } + + public String[] getValues() { + return ZERO_TO_NINETEEN; + } + + /** + * @param target + * the target to process + * @return the number of accented syllables since the last major break + */ + public byte process(Target target) { + Element segment = target.getMaryxmlElement(); + if (segment == null) + return 0; + Element phrase = (Element) MaryDomUtils.getAncestor(segment, MaryXML.PHRASE); + if (phrase == null) + return 0; + int count = 0; + TreeWalker tw = MaryDomUtils.createTreeWalker(phrase, MaryXML.SYLLABLE); + Element syllable = (Element) MaryDomUtils.getAncestor(segment, MaryXML.SYLLABLE); + if (syllable != null) { + tw.setCurrentNode(syllable); + } else { + tw.setCurrentNode(segment); + } + Element e; + while ((e = (Element) tw.previousNode()) != null && count < RAIL_LIMIT) { + String accent = e.getAttribute("accent"); + if (!accent.equals("")) + count++; + } + return (byte) count; + } + } + + /** + * Counts the number of accented syllables until the end of the phrase. + */ + public static class AccentedSylsFromPhraseEnd implements ByteValuedFeatureProcessor { + public AccentedSylsFromPhraseEnd() { + } + + public String getName() { + return "accented_syls_from_phrase_end"; + } + + public String[] getValues() { + return ZERO_TO_NINETEEN; + } + + /** + * @param target + * the target to process + * @return the number of accented syllables since the last major break + */ + public byte process(Target target) { + Element segment = target.getMaryxmlElement(); + if (segment == null) + return 0; + Element phrase = (Element) MaryDomUtils.getAncestor(segment, MaryXML.PHRASE); + if (phrase == null) + return 0; + int count = 0; + TreeWalker tw = MaryDomUtils.createTreeWalker(phrase, MaryXML.SYLLABLE); + tw.setCurrentNode(segment); + Element e; + while ((e = (Element) tw.nextNode()) != null && count < RAIL_LIMIT) { + String accent = e.getAttribute("accent"); + if (!accent.equals("")) + count++; + } + return (byte) count; + } + } + + /** + * Counts the number of words since the start of the phrase. + */ + public static class WordsFromPhraseStart implements ByteValuedFeatureProcessor { + public WordsFromPhraseStart() { + } + + public String getName() { + return "words_from_phrase_start"; + } + + public String[] getValues() { + return ZERO_TO_NINETEEN; + } + + /** + * @param target + * the target to process + * @return the number of words since the last major break + */ + public byte process(Target target) { + Element segment = target.getMaryxmlElement(); + if (segment == null) + return 0; + Element phrase = (Element) MaryDomUtils.getAncestor(segment, MaryXML.PHRASE); + if (phrase == null) + return 0; + int count = 0; + TreeWalker tw = MaryDomUtils.createTreeWalker(phrase, MaryXML.TOKEN); + Element word = (Element) MaryDomUtils.getAncestor(segment, MaryXML.TOKEN); + if (word != null) { + tw.setCurrentNode(word); + } else { + tw.setCurrentNode(segment); + } + Element e; + while ((e = (Element) tw.previousNode()) != null && count < RAIL_LIMIT) { + // only count tokens that have a "ph" attribute: + if (e.hasAttribute("ph")) + count++; + } + return (byte) count; + } + } + + /** + * Counts the number of words until the end of the phrase. + */ + public static class WordsFromPhraseEnd implements ByteValuedFeatureProcessor { + public WordsFromPhraseEnd() { + } + + public String getName() { + return "words_from_phrase_end"; + } + + public String[] getValues() { + return ZERO_TO_NINETEEN; + } + + /** + * @param target + * the target to process + * @return the number of words until the next major break + */ + public byte process(Target target) { + Element segment = target.getMaryxmlElement(); + if (segment == null) + return 0; + Element phrase = (Element) MaryDomUtils.getAncestor(segment, MaryXML.PHRASE); + if (phrase == null) + return 0; + int count = 0; + TreeWalker tw = MaryDomUtils.createTreeWalker(phrase, MaryXML.TOKEN); + tw.setCurrentNode(segment); + Element e; + while ((e = (Element) tw.nextNode()) != null && count < RAIL_LIMIT) { + // only count tokens that have a "ph" attribute + if (e.hasAttribute("ph")) + count++; + } + return (byte) count; + } + } + + /** + * Counts the number of words since the start of the sentence. + */ + public static class WordsFromSentenceStart implements ByteValuedFeatureProcessor { + public WordsFromSentenceStart() { + } + + public String getName() { + return "words_from_sentence_start"; + } + + public String[] getValues() { + return ZERO_TO_NINETEEN; + } + + /** + * @param target + * the target to process + * @return the number of words since the beginning of the sentence + */ + public byte process(Target target) { + Element segment = target.getMaryxmlElement(); + if (segment == null) + return 0; + Element sentence = (Element) MaryDomUtils.getAncestor(segment, MaryXML.SENTENCE); + if (sentence == null) + return 0; + int count = 0; + TreeWalker tw = MaryDomUtils.createTreeWalker(sentence, MaryXML.TOKEN); + Element word = (Element) MaryDomUtils.getAncestor(segment, MaryXML.TOKEN); + if (word != null) { + tw.setCurrentNode(word); + } else { + tw.setCurrentNode(segment); + } + Element e; + while ((e = (Element) tw.previousNode()) != null && count < RAIL_LIMIT) { + // only count tokens that have a "ph" attribute: + if (e.hasAttribute("ph")) + count++; + } + return (byte) count; + } + } + + /** + * Counts the number of words until the end of the sentence. + */ + public static class WordsFromSentenceEnd implements ByteValuedFeatureProcessor { + public WordsFromSentenceEnd() { + } + + public String getName() { + return "words_from_sentence_end"; + } + + public String[] getValues() { + return ZERO_TO_NINETEEN; + } + + /** + * @param target + * the target to process + * @return the number of words until the end of the sentence + */ + public byte process(Target target) { + Element segment = target.getMaryxmlElement(); + if (segment == null) + return 0; + Element sentence = (Element) MaryDomUtils.getAncestor(segment, MaryXML.SENTENCE); + if (sentence == null) + return 0; + int count = 0; + TreeWalker tw = MaryDomUtils.createTreeWalker(sentence, MaryXML.TOKEN); + tw.setCurrentNode(segment); + Element e; + while ((e = (Element) tw.nextNode()) != null && count < RAIL_LIMIT) { + // only count tokens that have a "ph" attribute: + if (e.hasAttribute("ph")) + count++; + } + return (byte) count; + } + } + + /** + * Counts the number of phrases since the start of the sentence. + */ + public static class PhrasesFromSentenceStart implements ByteValuedFeatureProcessor { + public PhrasesFromSentenceStart() { + } + + public String getName() { + return "phrases_from_sentence_start"; + } + + public String[] getValues() { + return ZERO_TO_NINETEEN; + } + + /** + * @param target + * the target to process + * @return the number of phrases since the start of the sentence + */ + public byte process(Target target) { + Element segment = target.getMaryxmlElement(); + if (segment == null) + return 0; + Element sentence = (Element) MaryDomUtils.getAncestor(segment, MaryXML.SENTENCE); + if (sentence == null) + return 0; + int count = 0; + TreeWalker tw = MaryDomUtils.createTreeWalker(sentence, MaryXML.PHRASE); + Element phrase = (Element) MaryDomUtils.getAncestor(segment, MaryXML.PHRASE); + if (phrase != null) { + tw.setCurrentNode(phrase); + } else { + tw.setCurrentNode(segment); + } + Element e; + while ((e = (Element) tw.previousNode()) != null && count < RAIL_LIMIT) { + count++; + } + return (byte) count; + } + } + + /** + * Counts the number of phrases until the end of the sentence. + */ + public static class PhrasesFromSentenceEnd implements ByteValuedFeatureProcessor { + public PhrasesFromSentenceEnd() { + } + + public String getName() { + return "phrases_from_sentence_end"; + } + + public String[] getValues() { + return ZERO_TO_NINETEEN; + } + + /** + * @param target + * the target to process + * @return the number of phrases until the end of the sentence. + */ + public byte process(Target target) { + Element segment = target.getMaryxmlElement(); + if (segment == null) + return 0; + Element sentence = (Element) MaryDomUtils.getAncestor(segment, MaryXML.SENTENCE); + if (sentence == null) + return 0; + int count = 0; + TreeWalker tw = MaryDomUtils.createTreeWalker(sentence, MaryXML.PHRASE); + tw.setCurrentNode(segment); + Element e; + while ((e = (Element) tw.nextNode()) != null && count < RAIL_LIMIT) { + count++; + } + return (byte) count; + } + } + + /** + * Counts the number of syllables since the last accent in the current phrase. + */ + public static class SylsFromPrevAccent implements ByteValuedFeatureProcessor { + public SylsFromPrevAccent() { + } + + public String getName() { + return "syls_from_prev_accent"; + } + + public String[] getValues() { + return ZERO_TO_NINETEEN; + } + + /** + * @param target + * the target to process + * @return the number of syllables since the last accent + */ + public byte process(Target target) { + Element segment = target.getMaryxmlElement(); + if (segment == null) + return 0; + Element phrase = (Element) MaryDomUtils.getAncestor(segment, MaryXML.PHRASE); + if (phrase == null) + return 0; + int count = 0; + TreeWalker tw = MaryDomUtils.createTreeWalker(phrase, MaryXML.SYLLABLE); + Element syllable = (Element) MaryDomUtils.getAncestor(segment, MaryXML.SYLLABLE); + if (syllable != null) { + tw.setCurrentNode(syllable); + } else { + tw.setCurrentNode(segment); + } + Element e; + while ((e = (Element) tw.previousNode()) != null && count < RAIL_LIMIT) { + count++; + String accent = e.getAttribute("accent"); + if (!accent.equals("")) + break; + } + return (byte) count; + } + } + + /** + * Counts the number of syllables until the next accent in the current phrase. + */ + public static class SylsToNextAccent implements ByteValuedFeatureProcessor { + public SylsToNextAccent() { + } + + public String getName() { + return "syls_to_next_accent"; + } + + public String[] getValues() { + return ZERO_TO_NINETEEN; + } + + /** + * @param target + * the target to process + * @return the number of syllables until the next accent + */ + public byte process(Target target) { + Element segment = target.getMaryxmlElement(); + if (segment == null) + return 0; + Element phrase = (Element) MaryDomUtils.getAncestor(segment, MaryXML.PHRASE); + if (phrase == null) + return 0; + int count = 0; + TreeWalker tw = MaryDomUtils.createTreeWalker(phrase, MaryXML.SYLLABLE); + tw.setCurrentNode(segment); + Element e; + while ((e = (Element) tw.nextNode()) != null && count < RAIL_LIMIT) { + count++; + String accent = e.getAttribute("accent"); + if (!accent.equals("")) + break; + } + return (byte) count; + } + } + + /** + * Counts the number of syllables since the last stressed syllable in the current phrase. + */ + public static class SylsFromPrevStressed implements ByteValuedFeatureProcessor { + public SylsFromPrevStressed() { + } + + public String getName() { + return "syls_from_prev_stressed"; + } + + public String[] getValues() { + return ZERO_TO_NINETEEN; + } + + /** + * @param target + * the target to process + * @return the number of syllables since the last accent + */ + public byte process(Target target) { + Element segment = target.getMaryxmlElement(); + if (segment == null) + return 0; + Element phrase = (Element) MaryDomUtils.getAncestor(segment, MaryXML.PHRASE); + if (phrase == null) + return 0; + int count = 0; + TreeWalker tw = MaryDomUtils.createTreeWalker(phrase, MaryXML.SYLLABLE); + Element syllable = (Element) MaryDomUtils.getAncestor(segment, MaryXML.SYLLABLE); + if (syllable != null) { + tw.setCurrentNode(syllable); + } else { + tw.setCurrentNode(segment); + } + Element e; + while ((e = (Element) tw.previousNode()) != null && count < RAIL_LIMIT) { + count++; + String stress = e.getAttribute("stress"); + if (stress.equals("1")) + break; + } + return (byte) count; + } + } + + /** + * Counts the number of syllables until the next stressed syllable in the current phrase. + */ + public static class SylsToNextStressed implements ByteValuedFeatureProcessor { + public SylsToNextStressed() { + } + + public String getName() { + return "syls_to_next_stressed"; + } + + public String[] getValues() { + return ZERO_TO_NINETEEN; + } + + /** + * @param target + * the target to process + * @return the number of syllables until the next stressed syllable + */ + public byte process(Target target) { + Element segment = target.getMaryxmlElement(); + if (segment == null) + return 0; + Element phrase = (Element) MaryDomUtils.getAncestor(segment, MaryXML.PHRASE); + if (phrase == null) + return 0; + int count = 0; + TreeWalker tw = MaryDomUtils.createTreeWalker(phrase, MaryXML.SYLLABLE); + tw.setCurrentNode(segment); + Element e; + while ((e = (Element) tw.nextNode()) != null && count < RAIL_LIMIT) { + count++; + String stress = e.getAttribute("stress"); + if (stress.equals("1")) + break; + } + return (byte) count; + } + } + + /** + * Determines the word punctuation. + */ + public static class WordPunc implements ByteValuedFeatureProcessor { + protected String name; + protected TargetElementNavigator navigator; + protected ByteStringTranslator values; + + /** + * @param name + * name of this feature processor + * @param wordNavigator + * a navigator which returns a word for a target. This navigator decides the word for which the punctuation + * will be computed. + */ + public WordPunc(String name, TargetElementNavigator wordNavigator) { + this.name = name; + this.navigator = wordNavigator; + this.values = new ByteStringTranslator(new String[] { "0", ".", ",", ";", ":", "(", ")", "?", "!", "\"" }); + } + + public String getName() { + return name; + } + + public String[] getValues() { + return values.getStringValues(); + } + + public byte process(Target target) { + Element word = navigator.getElement(target); + if (word == null) + return 0; + Element sentence = (Element) MaryDomUtils.getAncestor(word, MaryXML.SENTENCE); + if (sentence == null) + return 0; + TreeWalker tw = MaryDomUtils.createTreeWalker(sentence, MaryXML.TOKEN, MaryXML.BOUNDARY); + tw.setCurrentNode(word); + Element next = (Element) tw.nextNode(); + if (next == null || !next.getTagName().equals(MaryXML.TOKEN) || next.hasAttribute("ph")) + return 0; + String text = MaryDomUtils.tokenText(next); + if (values.contains(text)) { + return values.get(text); + } + // unknown or no punctuation: return "0" + return values.get("0"); + } + } + + /** + * Determines the next word punctuation in the sentence. + */ + public static class NextPunctuation extends WordPunc { + public NextPunctuation() { + super("next_punctuation", new WordNavigator()); + } + + public byte process(Target target) { + Element word = navigator.getElement(target); + if (word == null) + return 0; + Element sentence = (Element) MaryDomUtils.getAncestor(word, MaryXML.SENTENCE); + if (sentence == null) + return 0; + TreeWalker tw = MaryDomUtils.createTreeWalker(sentence, MaryXML.TOKEN); + tw.setCurrentNode(word); + Element e; + while ((e = (Element) tw.nextNode()) != null) { + if (e.hasAttribute("ph")) // a word + continue; + // potentially a punctuation + String text = MaryDomUtils.tokenText(e); + if (values.contains(text)) { + return values.get(text); + } + } + // no next punctuation: return "0" + return values.get("0"); + } + } + + /** + * Determines the previous word punctuation in the sentence. + */ + public static class PrevPunctuation extends WordPunc { + public PrevPunctuation() { + super("prev_punctuation", new WordNavigator()); + } + + public byte process(Target target) { + Element word = navigator.getElement(target); + if (word == null) + return 0; + Element sentence = (Element) MaryDomUtils.getAncestor(word, MaryXML.SENTENCE); + if (sentence == null) + return 0; + TreeWalker tw = MaryDomUtils.createTreeWalker(sentence, MaryXML.TOKEN); + tw.setCurrentNode(word); + Element e; + while ((e = (Element) tw.previousNode()) != null) { + if (e.hasAttribute("ph")) // a word + continue; + // potentially a punctuation + String text = MaryDomUtils.tokenText(e); + if (values.contains(text)) { + return values.get(text); + } + } + // no next punctuation: return "0" + return values.get("0"); + } + } + + /** + * Determines the distance in words to the next word punctuation in the sentence. + */ + public static class WordsToNextPunctuation extends WordPunc { + public WordsToNextPunctuation() { + super("words_to_next_punctuation", new WordNavigator()); + } + + public String[] getValues() { + return ZERO_TO_NINETEEN; + } + + public byte process(Target target) { + Element word = navigator.getElement(target); + if (word == null) + return 0; + Element sentence = (Element) MaryDomUtils.getAncestor(word, MaryXML.SENTENCE); + if (sentence == null) + return 0; + TreeWalker tw = MaryDomUtils.createTreeWalker(sentence, MaryXML.TOKEN); + tw.setCurrentNode(word); + Element e; + int count = 0; + while ((e = (Element) tw.nextNode()) != null && count < RAIL_LIMIT) { + count++; + if (e.hasAttribute("ph")) // a word + continue; + // potentially a punctuation + String text = MaryDomUtils.tokenText(e); + if (values.contains(text)) { + break; + } + } + // found punctuation or end of sentence: + return (byte) count; + } + } + + /** + * Determines the distance in words from the previous word punctuation in the sentence. + */ + public static class WordsFromPrevPunctuation extends WordPunc { + public WordsFromPrevPunctuation() { + super("words_from_prev_punctuation", new WordNavigator()); + } + + public String[] getValues() { + return ZERO_TO_NINETEEN; + } + + public byte process(Target target) { + Element word = navigator.getElement(target); + if (word == null) + return 0; + Element sentence = (Element) MaryDomUtils.getAncestor(word, MaryXML.SENTENCE); + if (sentence == null) + return 0; + TreeWalker tw = MaryDomUtils.createTreeWalker(sentence, MaryXML.TOKEN); + tw.setCurrentNode(word); + Element e; + int count = 0; + while ((e = (Element) tw.previousNode()) != null && count < RAIL_LIMIT) { + count++; + if (e.hasAttribute("ph")) // a word + continue; + // potentially a punctuation + String text = MaryDomUtils.tokenText(e); + if (values.contains(text)) { + break; + } + } + // found punctuation or start of sentence: + return (byte) count; + } + } + + /** + * Determine the prosodic property of a target + * + * @author Anna Hunecke + * + */ + public static class Selection_Prosody implements ByteValuedFeatureProcessor { + + protected TargetElementNavigator navigator; + protected ByteStringTranslator values = new ByteStringTranslator(new String[] { "0", "stressed", "pre-nuclear", + "nuclear", "finalHigh", "finalLow", "final" }); + private Set lowEndtones = new HashSet(Arrays.asList(new String[] { "L-", "L-%", "L-L%" })); + private Set highEndtones = new HashSet(Arrays.asList(new String[] { "H-", "!H-", "H-%", "H-L%", "!H-%", + "H-^H%", "!H-^H%", "L-H%", "H-H%" })); + + public Selection_Prosody(TargetElementNavigator syllableNavigator) { + this.navigator = syllableNavigator; + } + + public String getName() { + return "selection_prosody"; + } + + public String[] getValues() { + return values.getStringValues(); + } + + /** + * Determine the prosodic property of the target + * + * @param target + * the target + * @return 0 - unstressed, 1 - stressed, 2 - pre-nuclear accent 3 - nuclear accent, 4 - phrase final high, 5 - phrase + * final low, 6 - phrase final (with unknown high/low status). + */ + public byte process(Target target) { + // first find out if syllable is stressed + Element syllable = navigator.getElement(target); + if (syllable == null) + return (byte) 0; + boolean stressed = false; + if (syllable.getAttribute("stress").equals("1")) { + stressed = true; + } + // find out if we have an accent + boolean accented = syllable.hasAttribute("accent"); + boolean nuclear = true; // relevant only if accented == true + // find out the position of the target + boolean phraseFinal = false; + String endtone = null; + Element sentence = (Element) MaryDomUtils.getAncestor(syllable, MaryXML.SENTENCE); + if (sentence == null) + return 0; + TreeWalker tw = MaryDomUtils.createTreeWalker(sentence, MaryXML.SYLLABLE, MaryXML.BOUNDARY); + tw.setCurrentNode(syllable); + Element e = (Element) tw.nextNode(); + if (e != null) { + if (e.getTagName().equals(MaryXML.BOUNDARY)) { + phraseFinal = true; + endtone = e.getAttribute("tone"); + } + if (accented) { // look forward for any accent + while (e != null) { + if (e.getTagName().equals(MaryXML.SYLLABLE) && e.hasAttribute("accent")) { + nuclear = false; + break; + } + e = (Element) tw.nextNode(); + } + } + } + // Now, we know: + // stressed or not + // accented or not + // if accented, nuclear or not + // if final, the endtone + + if (accented) { + if (nuclear) { + return values.get("nuclear"); + } else { + return values.get("pre-nuclear"); + } + } else if (phraseFinal) { + if (endtone != null && highEndtones.contains(endtone)) { + return values.get("finalHigh"); + } else if (endtone != null && lowEndtones.contains(endtone)) { + return values.get("finalLow"); + } else { + return values.get("final"); + } + } else if (stressed) { + return values.get("stressed"); + } + return (byte) 0;// return unstressed + } + } + + /** + * Returns the duration of the given segment, in seconds. + */ + public static class UnitDuration implements ContinuousFeatureProcessor { + public String getName() { + return "unit_duration"; + } + + public float process(Target target) { + if (target instanceof DiphoneTarget) { + DiphoneTarget diphone = (DiphoneTarget) target; + return process(diphone.left) + process(diphone.right); + } + Element seg = target.getMaryxmlElement(); + if (seg == null) { + return 0; + } + float phoneDuration = 0; + String sDur; + if (seg.getTagName().equals(MaryXML.PHONE)) + sDur = seg.getAttribute("d"); + else { + assert seg.getTagName().equals(MaryXML.BOUNDARY) : "segment should be a phone or a boundary, but is a " + + seg.getTagName(); + sDur = seg.getAttribute("duration"); + } + if (sDur.equals("")) { + return 0; + } + try { + // parse duration string, and convert from milliseconds into seconds: + phoneDuration = Float.parseFloat(sDur) * 0.001f; + } catch (NumberFormatException nfe) { + } + if (target instanceof HalfPhoneTarget) + return phoneDuration / 2; + return phoneDuration; + } + } + + /** + * Calculates the log of the fundamental frequency in the middle of a unit segment. This processor should be used by target + * items only -- for unit features during voice building, the actual measured values should be used. + */ + public static class UnitLogF0 implements ContinuousFeatureProcessor { + public String getName() { + return "unit_logf0"; + } + + public float process(Target target) { + return process(target, false); + } + + /** + * Compute log f0 and log f0 delta for the given target. + * + * @param target + * @param delta + * if true, return the delta, i.e. the logF0 slope; if false, return the log f0 value itself. + * @return + */ + protected float process(Target target, boolean delta) { + // Note: all variables in this method with "f0" in their name + // actually represent log f0 values. + if (target instanceof DiphoneTarget) { + DiphoneTarget diphone = (DiphoneTarget) target; + return (process(diphone.left) + process(diphone.right)) / 2; + } + // Idea: find the closest f0 targets in the current syllable, left and right of our middle; + // linearly interpolate between them to find the value in the middle of this unit. + Element seg = target.getMaryxmlElement(); + if (seg == null) { + return 0; + } + if (!seg.getTagName().equals(MaryXML.PHONE)) { + return 0; + } + // get mid position of segment wrt phone start (phone start = 0, phone end = phone duration) + float mid; + float phoneDuration = getDuration(seg); + if (target instanceof HalfPhoneTarget) { + if (((HalfPhoneTarget) target).isLeftHalf()) { + mid = .25f; + } else { + mid = .75f; + } + } else { // phone target + mid = .5f; + } + + // Now mid is the middle of the unit relative to the phone start, in percent + Float lastPos = null; // position relative to mid, in milliseconds (negative) + float lastF0 = 0; + Float nextPos = null; // position relative to mid, in milliseconds + float nextF0 = 0; + Float[] f0values = getLogF0Values(seg); + + assert f0values != null; + // values are position, f0, position, f0, etc.; + // position is in percent of phone duration between 0 and 1, f0 is in Hz + for (int i = 0; i < f0values.length; i += 2) { + float pos = f0values[i]; + if (pos <= mid) { + lastPos = (pos - mid) * phoneDuration; // negative or zero + lastF0 = f0values[i + 1]; + } else if (pos > mid) { + nextPos = (pos - mid) * phoneDuration; // positive + nextF0 = f0values[i + 1]; + break; // no point looking further to the right + } + } + if (lastPos == null) { // need to look to the left + float msBack = -mid * phoneDuration; + Element e = seg; + + // get all phone units in the same phrase + Element phraseElement = (Element) MaryDomUtils.getAncestor(seg, MaryXML.PHRASE); + TreeWalker tw = MaryDomUtils.createTreeWalker(seg.getOwnerDocument(), phraseElement, MaryXML.PHONE); + Element en; + while ((en = (Element) tw.nextNode()) != null) { + if (en == seg) { + break; + } + } + + while ((e = (Element) tw.previousNode()) != null) { + float dur = getDuration(e); + f0values = getLogF0Values(e); + if (f0values.length == 0) { + msBack -= dur; + continue; + } + assert f0values.length > 1; + float pos = f0values[f0values.length - 2]; + lastPos = msBack - (1 - pos) * dur; + lastF0 = f0values[f0values.length - 1]; + break; + } + } + + if (nextPos == null) { // need to look to the right + float msForward = (1 - mid) * phoneDuration; + Element e = seg; + + // get all phone units in the same phrase + Element phraseElement = (Element) MaryDomUtils.getAncestor(seg, MaryXML.PHRASE); + TreeWalker tw = MaryDomUtils.createTreeWalker(seg.getOwnerDocument(), phraseElement, MaryXML.PHONE); + Element en; + while ((en = (Element) tw.nextNode()) != null) { + if (en == seg) { + break; + } + } + + while ((e = (Element) tw.nextNode()) != null) { + float dur = getDuration(e); + f0values = getLogF0Values(e); + if (f0values.length == 0) { + msForward += dur; + continue; + } + assert f0values.length > 1; + float pos = f0values[0]; + nextPos = msForward + pos * dur; + nextF0 = f0values[1]; + break; + } + } + + if (lastPos == null && nextPos == null) { + // no info + return 0; + } else if (lastPos == null) { + // have only nextF0; + if (delta) + return 0; + else + return nextF0; + } else if (nextPos == null) { + // have only lastF0 + if (delta) + return 0; + else + return lastF0; + } + assert lastPos <= 0 && 0 <= nextPos : "unexpected: lastPos=" + lastPos + ", nextPos=" + nextPos; + // build a linear function (f(x) = slope*x+intersectionYAxis) + float f0; + float slope; + if (lastPos - nextPos == 0) { + f0 = (lastF0 + nextF0) / 2; + slope = 0; + } else { + slope = (nextF0 - lastF0) / (nextPos - lastPos); + // calculate the pitch + f0 = lastF0 + slope * (-lastPos); + } + assert !Float.isNaN(f0) : "f0 is not a number"; + assert lastF0 <= f0 && nextF0 >= f0 || lastF0 >= f0 && nextF0 <= f0 : "f0 should be between last and next values"; + + if (delta) + return slope; + else + return f0; + } + + private Float[] getLogF0Values(Element ph) { + String mbrTargets = ph.getAttribute("f0"); + if (mbrTargets.equals("")) { + return new Float[0]; + } + ArrayList values = new ArrayList(); + try { + // mbrTargets contains one or more pairs of numbers, + // either enclosed by (a,b) or just separated by whitespace. + StringTokenizer st = new StringTokenizer(mbrTargets, " (,)"); + while (st.hasMoreTokens()) { + String posString = ""; + while (st.hasMoreTokens() && posString.equals("")) + posString = st.nextToken(); + String f0String = ""; + while (st.hasMoreTokens() && f0String.equals("")) + f0String = st.nextToken(); + + float pos = Float.parseFloat(posString) * 0.01f; + assert 0 <= pos && pos <= 1 : "invalid position:" + pos + " (pos string was '" + posString + + "' coming from '" + mbrTargets + "')"; + float f0 = Float.parseFloat(f0String); + float logF0 = (float) Math.log(f0); + values.add(pos); + values.add(logF0); + } + } catch (Exception e) { + return new Float[0]; + } + return values.toArray(new Float[0]); + } + + private float getDuration(Element ph) { + float phoneDuration = 0; + String sDur = ph.getAttribute("d"); + if (!sDur.equals("")) { + try { + phoneDuration = Float.parseFloat(sDur); + } catch (NumberFormatException nfe) { + } + } + return phoneDuration; + } + } + + /** + * Calculates the slope of a linear approximation of the fundamental frequency, in the log domain. The slope is computed by + * linearly connecting the two log f0 values closest to the middle of the unit segment. This processor should be used by + * target items only -- for unit features during voice building, the actual measured values should be used. + */ + public static class UnitLogF0Delta extends UnitLogF0 { + @Override + public String getName() { + return "unit_logf0delta"; + } + + public float process(Target target) { + return process(target, true); + } + } + + /** + * Returns the value of the given feature for the given segment. + */ + public static class GenericContinuousFeature implements ContinuousFeatureProcessor { + private String name; + private String attributeName; + + public GenericContinuousFeature(String featureName, String attributeName) { + this.name = featureName; + this.attributeName = attributeName; + } + + public String getName() { + return name; + } + + public float process(Target target) { + if (target instanceof DiphoneTarget) { + DiphoneTarget diphone = (DiphoneTarget) target; + // return mean of left and right costs: + return (process(diphone.left) + process(diphone.right)) / 2.0f; + } + Element seg = target.getMaryxmlElement(); + if (seg == null) { + return 0; + } + float value = 0; + String valueString; + if (seg.getTagName().equals(MaryXML.PHONE)) { + valueString = seg.getAttribute(attributeName); + } else { + assert seg.getTagName().equals(MaryXML.BOUNDARY) : "segment should be a phone or a boundary, but is a " + + seg.getTagName(); + valueString = seg.getAttribute(attributeName); + } + if (valueString.equals("")) { + return 0; + } + try { + value = Float.parseFloat(valueString); + } catch (NumberFormatException nfe) { + return 0; + } + return value; + } + } + +} diff --git a/marytts-unitselection/src/main/java/marytts/features/MaryLanguageFeatureProcessors.java b/marytts-unitselection/src/main/java/marytts/features/MaryLanguageFeatureProcessors.java new file mode 100644 index 0000000000..5813f90c61 --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/features/MaryLanguageFeatureProcessors.java @@ -0,0 +1,468 @@ +/** + * Portions Copyright 2006 DFKI GmbH. + * Portions Copyright 2001 Sun Microsystems, Inc. + * Portions Copyright 1999-2001 Language Technologies Institute, + * Carnegie Mellon University. + * All Rights Reserved. Use is subject to license terms. + * + * Permission is hereby granted, free of charge, to use and distribute + * this software and its documentation without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of this work, and to + * permit persons to whom this work is furnished to do so, subject to + * the following conditions: + * + * 1. The code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * 2. Any modifications must be clearly marked as such. + * 3. Original authors' names are not deleted. + * 4. The authors' names are not used to endorse or promote products + * derived from this software without specific prior written + * permission. + * + * DFKI GMBH AND THE CONTRIBUTORS TO THIS WORK DISCLAIM ALL WARRANTIES WITH + * REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL DFKI GMBH NOR THE + * CONTRIBUTORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL + * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR + * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS + * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF + * THIS SOFTWARE. + */ +package marytts.features; + +import java.io.InputStream; +import java.util.Map; + +import marytts.datatypes.MaryXML; +import marytts.fst.FSTLookup; +import marytts.modules.phonemiser.AllophoneSet; +import marytts.util.dom.MaryDomUtils; +import marytts.util.string.ByteStringTranslator; + +import org.w3c.dom.Element; + +/** + * Provides the set of feature processors that are used by this language as part of the CART processing. + */ +public class MaryLanguageFeatureProcessors extends MaryGenericFeatureProcessors { + + // no instances + private MaryLanguageFeatureProcessors() { + } + + /** + * The phone symbol for the given target. + * + * @author Marc Schröder + * + */ + public static class Phone implements ByteValuedFeatureProcessor { + protected String name; + protected ByteStringTranslator values; + protected String pauseSymbol; + protected TargetElementNavigator navigator; + + /** + * Initialise a phone feature processor. + * + * @param name + * the name of the feature + * @param possibleValues + * the list of possible phone values for the phonetic alphabet used, plus the value "0"=n/a. + * @param segmentNavigator + * a navigator returning a segment with respect to the target. + */ + public Phone(String name, String[] possibleValues, String pauseSymbol, TargetElementNavigator segmentNavigator) { + this.name = name; + this.values = new ByteStringTranslator(possibleValues); + this.pauseSymbol = pauseSymbol; + this.navigator = segmentNavigator; + } + + public String getName() { + return name; + } + + public String[] getValues() { + return values.getStringValues(); + } + + public byte process(Target target) { + Element segment = navigator.getElement(target); + if (segment == null) + return values.get(pauseSymbol); + if (!segment.getTagName().equals(MaryXML.PHONE)) + return values.get(pauseSymbol); + String ph = segment.getAttribute("p"); + if (!values.contains(ph)) + return values.get("0"); + return values.get(ph); + } + + public String getPauseSymbol() { + return pauseSymbol; + } + } + + /** + * The unit name for the given half phone target. + * + * @author Marc Schröder + * + */ + public static class HalfPhoneUnitName implements ByteValuedFeatureProcessor { + protected String name; + protected ByteStringTranslator values; + protected String pauseSymbol; + + /** + * Initialise a UnitName feature processor. + * + * @param name + * the name of the feature + * @param phoneset + * the phonetic alphabet used + * @param segmentNavigator + * a navigator returning a segment with respect to the target. + */ + public HalfPhoneUnitName(String[] possiblePhonemes, String pauseSymbol) { + this.name = "halfphone_unitname"; + this.pauseSymbol = pauseSymbol; + String[] possibleValues = new String[2 * possiblePhonemes.length + 1]; + possibleValues[0] = "0"; // the "n/a" value + for (int i = 0; i < possiblePhonemes.length; i++) { + possibleValues[2 * i + 1] = possiblePhonemes[i] + "_L"; + possibleValues[2 * i + 2] = possiblePhonemes[i] + "_R"; + } + this.values = new ByteStringTranslator(possibleValues); + } + + public String getName() { + return name; + } + + public String[] getValues() { + return values.getStringValues(); + } + + public byte process(Target target) { + if (!(target instanceof HalfPhoneTarget)) + return 0; + HalfPhoneTarget hpTarget = (HalfPhoneTarget) target; + Element segment = target.getMaryxmlElement(); + String phoneLabel; + if (segment == null) { + phoneLabel = pauseSymbol; + } else if (!segment.getTagName().equals(MaryXML.PHONE)) { + phoneLabel = pauseSymbol; + } else { + phoneLabel = segment.getAttribute("p"); + } + if (phoneLabel.equals("")) + return values.get("0"); + String unitLabel = phoneLabel + (hpTarget.isLeftHalf() ? "_L" : "_R"); + return values.get(unitLabel); + } + } + + /** + * A parametrisable class which can retrieve all sorts of phone features, given a phone set. + * + * @author Marc Schröder + * + */ + public static class PhoneFeature implements ByteValuedFeatureProcessor { + protected AllophoneSet phoneSet; + protected String name; + protected String phonesetQuery; + protected ByteStringTranslator values; + protected String pauseSymbol; + protected TargetElementNavigator navigator; + + public PhoneFeature(AllophoneSet phoneSet, String name, String phonesetQuery, String[] possibleValues, + String pauseSymbol, TargetElementNavigator segmentNavigator) { + this.phoneSet = phoneSet; + this.name = name; + this.phonesetQuery = phonesetQuery; + this.values = new ByteStringTranslator(possibleValues); + this.navigator = segmentNavigator; + } + + public String getName() { + return name; + } + + public String[] getValues() { + return values.getStringValues(); + } + + public byte process(Target target) { + Element segment = navigator.getElement(target); + if (segment == null) + return values.get("0"); + String ph; + if (!segment.getTagName().equals(MaryXML.PHONE)) { + ph = pauseSymbol; + } else { + ph = segment.getAttribute("p"); + } + String value = phoneSet.getPhoneFeature(ph, phonesetQuery); + if (value == null) + return values.get("0"); + return values.get(value); + } + } + + /** + * Returns the part-of-speech. + */ + public static class Pos implements ByteValuedFeatureProcessor { + private ByteStringTranslator values; + private TargetElementNavigator navigator; + private String name; + + public String getName() { + return this.name; + } + + public String[] getValues() { + return values.getStringValues(); + } + + public Pos(String[] posValues) { + this.values = new ByteStringTranslator(posValues); + this.navigator = new WordNavigator(); + this.name = "pos"; + } + + public Pos(String aName, String[] posValues, TargetElementNavigator wordNavi) { + this.values = new ByteStringTranslator(posValues); + this.navigator = wordNavi; + this.name = aName; + } + + /** + * @param item + * the item to process + * @return a guess at the part-of-speech for the item + */ + public byte process(Target target) { + Element word = navigator.getElement(target); + if (word == null) + return values.get("0"); + String pos = word.getAttribute("pos"); + if (pos == null) + return values.get("0"); + pos = pos.trim(); + if (values.contains(pos)) + return values.get(pos); + return values.get("0"); + } + } + + /** + * Returns generalised part-of-speech. + */ + public static class Gpos implements ByteValuedFeatureProcessor { + private Map posConverter; + private ByteStringTranslator values; + private TargetElementNavigator navigator; + + public String getName() { + return "gpos"; + } + + public String[] getValues() { + return values.getStringValues(); + } + + public Gpos(Map posConverter) { + this.posConverter = posConverter; + this.values = new ByteStringTranslator(new String[] { "0", "in", // Preposition or subordinating conjunction + "to", // to + "det", // determiner + "md", // modal + "cc", // coordinating conjunction + "wp", // w-pronouns + "pps", // possive pronouns + "aux", // auxiliary verbs + "punc", // punctuation + "content" // content words + }); + this.navigator = new WordNavigator(); + } + + /** + * Performs some processing on the given item. + * + * @param item + * the item to process + * @return a guess at the part-of-speech for the item + */ + public byte process(Target target) { + Element word = navigator.getElement(target); + if (word == null) + return values.get("0"); + String pos = word.getAttribute("pos"); + if (pos == null) + return values.get("0"); + pos = pos.trim(); + if (posConverter.containsKey(pos)) { + pos = (String) posConverter.get(pos); + } + if (!values.contains(pos)) + return values.get("0"); + return values.get(pos); + } + } + + /** + * Checks for onset coda This is a feature processor. A feature processor takes an item, performs some sort of processing on + * the item and returns an object. + */ + public static class SegOnsetCoda implements ByteValuedFeatureProcessor { + protected ByteStringTranslator values; + private AllophoneSet phoneSet; + + public SegOnsetCoda(AllophoneSet phoneSet) { + this.phoneSet = phoneSet; + this.values = new ByteStringTranslator(new String[] { "0", "onset", "coda" }); + } + + public String getName() { + return "onsetcoda"; + } + + public String[] getValues() { + return values.getStringValues(); + } + + public byte process(Target target) { + Element s = target.getMaryxmlElement(); + if (s == null) { + return 0; + } + if (!s.getTagName().equals(MaryXML.PHONE)) + return 0; + + while ((s = MaryDomUtils.getNextSiblingElement(s)) != null) { + String ph = s.getAttribute("p"); + if ("+".equals(phoneSet.getPhoneFeature(ph, "vc"))) { + return values.get("onset"); + } + } + return values.get("coda"); + } + } + + /** + * The phone class for the given target. + * + * @author Anna Hunecke + * + */ + public static class Selection_PhoneClass implements ByteValuedFeatureProcessor { + protected String name; + protected Map phones2Classes; + protected ByteStringTranslator values; + protected TargetElementNavigator navigator; + + /** + * Initialise the feature processor. + * + * @param phones2Classes + * the mapping of phones to their classes + * @param classes + * the available phone classes + * @param segmentNavigator + * a navigator returning a segment with respect to the target. + */ + public Selection_PhoneClass(Map phones2Classes, String[] classes, TargetElementNavigator segmentNavigator) { + this.name = "selection_next_phone_class"; + this.phones2Classes = phones2Classes; + this.values = new ByteStringTranslator(classes); + this.navigator = segmentNavigator; + } + + public String getName() { + return name; + } + + public String[] getValues() { + return values.getStringValues(); + } + + /** + * Give back the phone class of the target + * + * @param target + * @return the phone class of the target + */ + public byte process(Target target) { + Element segment = navigator.getElement(target); + if (segment == null) + return values.get("0"); + if (!segment.getTagName().equals(MaryXML.PHONE)) + return 0; + String ph = segment.getAttribute("p"); + String phoneClass = phones2Classes.get(ph); + if (phoneClass == null) { + return values.get("0"); + } + return values.get(phoneClass); + } + } + + public static class WordFrequency implements ByteValuedFeatureProcessor { + protected TargetElementNavigator navigator; + protected ByteStringTranslator values; + protected FSTLookup wordFrequencies; + + public WordFrequency(InputStream inStream, String identifier, String encoding) { + this.navigator = new WordNavigator(); + try { + if (inStream != null) + this.wordFrequencies = new FSTLookup(inStream, identifier, encoding); + else + this.wordFrequencies = null; + } catch (Exception e) { + throw new RuntimeException(e); + } + this.values = new ByteStringTranslator(new String[] { "0", "1", "2", "3", "4", "5", "6", "7", "8", "9" }); + } + + public String getName() { + return "word_frequency"; + } + + public String[] getValues() { + return values.getStringValues(); + } + + /** + * Performs some processing on the given item. + * + * @param target + * the target to process + * @return the frequency of the current word, on a ten-point scale from 0=unknown=very rare to 9=very frequent. + */ + public byte process(Target target) { + Element word = navigator.getElement(target); + if (word == null) + return (byte) 0; + String wordString = MaryDomUtils.tokenText(word); + if (wordFrequencies != null) { + String[] result = wordFrequencies.lookup(wordString); + if (result.length > 0) { + String freq = result[0]; + if (values.contains(freq)) + return values.get(freq); + } + + } + return (byte) 0; // unknown word + } + } + +} diff --git a/marytts-unitselection/src/main/java/marytts/features/ShortValuedFeatureProcessor.java b/marytts-unitselection/src/main/java/marytts/features/ShortValuedFeatureProcessor.java new file mode 100644 index 0000000000..bd36a1048b --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/features/ShortValuedFeatureProcessor.java @@ -0,0 +1,48 @@ +/** + * Portions Copyright 2006 DFKI GmbH. + * Portions Copyright 2001 Sun Microsystems, Inc. + * Portions Copyright 1999-2001 Language Technologies Institute, + * Carnegie Mellon University. + * All Rights Reserved. Use is subject to license terms. + * + * Permission is hereby granted, free of charge, to use and distribute + * this software and its documentation without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of this work, and to + * permit persons to whom this work is furnished to do so, subject to + * the following conditions: + * + * 1. The code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * 2. Any modifications must be clearly marked as such. + * 3. Original authors' names are not deleted. + * 4. The authors' names are not used to endorse or promote products + * derived from this software without specific prior written + * permission. + * + * DFKI GMBH AND THE CONTRIBUTORS TO THIS WORK DISCLAIM ALL WARRANTIES WITH + * REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL DFKI GMBH NOR THE + * CONTRIBUTORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL + * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR + * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS + * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF + * THIS SOFTWARE. + */ +package marytts.features; + + +/** + * Performs a specific type of processing on an item and returns an object. + */ +public interface ShortValuedFeatureProcessor extends MaryFeatureProcessor { + /** + * List the possible values of the feature processor, as clear-text values. Short values as returned by process() can be + * translated into their string equivalent by using the short value as an index in the String[] returned. + * + * @return an array containing the possible return values of this feature processor, in String representation. + */ + public String[] getValues(); + + public short process(Target target); +} diff --git a/marytts-unitselection/src/main/java/marytts/features/Target.java b/marytts-unitselection/src/main/java/marytts/features/Target.java new file mode 100644 index 0000000000..25486764dd --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/features/Target.java @@ -0,0 +1,191 @@ +/** + * Copyright 2006 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.features; + +import marytts.datatypes.MaryXML; +import marytts.features.FeatureVector; +import marytts.features.MaryGenericFeatureProcessors; +import marytts.modules.phonemiser.Allophone; +import marytts.modules.phonemiser.AllophoneSet; +import marytts.modules.synthesis.Voice; +import marytts.util.MaryRuntimeUtils; +import marytts.util.dom.MaryDomUtils; + +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.UserDataHandler; + +/** + * A representation of a target representing the ideal properties of a unit in a target utterance. + * + * @author Marc Schröder + * + */ +public class Target { + protected String name; + protected Element maryxmlElement; + + protected FeatureVector featureVector = null; + + protected float duration = -1; + protected float f0 = -1; + protected int isSilence = -1; + + /** + * Create a target associated to the given element in the MaryXML tree. + * + * @param name + * a name for the target, which may or may not coincide with the segment name. + * @param maryxmlElement + * the phone or boundary element in the MaryXML tree to be associated with this target. + */ + public Target(String name, Element maryxmlElement) { + this.name = name; + this.maryxmlElement = maryxmlElement; + } + + public Element getMaryxmlElement() { + return maryxmlElement; + } + + public String getName() { + return name; + } + + public FeatureVector getFeatureVector() { + return featureVector; + } + + public void setFeatureVector(FeatureVector featureVector) { + this.featureVector = featureVector; + } + + public float getTargetDurationInSeconds() { + if (duration != -1) { + return duration; + } else { + if (maryxmlElement == null) + return 0; + // throw new NullPointerException("Target "+name+" does not have a maryxml element."); + duration = new MaryGenericFeatureProcessors.UnitDuration().process(this); + return duration; + } + } + + /** + * adapted from {@link MaryGenericFeatureProcessors.UnitDuration#process} + * + * @param newDuration + */ + public void setTargetDurationInSeconds(float newDuration) { + if (maryxmlElement != null) { + if (maryxmlElement.getTagName().equals(MaryXML.PHONE)) { + maryxmlElement.setAttribute("d", Float.toString(newDuration)); + } else { + assert maryxmlElement.getTagName().equals(MaryXML.BOUNDARY) : "segment should be a phone or a boundary, but is a " + + maryxmlElement.getTagName(); + maryxmlElement.setAttribute("duration", Float.toString(newDuration)); + } + } + } + + public float getTargetF0InHz() { + if (f0 != -1) { + return f0; + } else { + if (maryxmlElement == null) + throw new NullPointerException("Target " + name + " does not have a maryxml element."); + float logf0 = new MaryGenericFeatureProcessors.UnitLogF0().process(this); + if (logf0 == 0) + f0 = 0; + else + f0 = (float) Math.exp(logf0); + return f0; + } + } + + public boolean hasFeatureVector() { + return featureVector != null; + } + + public static UserDataHandler targetFeatureCloner = new UserDataHandler() { + public void handle(short operation, String key, Object data, Node src, Node dest) { + if (operation == UserDataHandler.NODE_CLONED && key == "target") { + dest.setUserData(key, data, this); + System.err.println("yay"); + } else { + System.err.println("nay"); + } + } + }; + + /** + * Determine whether this target is a silence target + * + * @return true if the target represents silence, false otherwise + */ + public boolean isSilence() { + + if (isSilence == -1) { + // TODO: how do we know the silence symbol here? + String silenceSymbol = "_"; + if (name.startsWith(silenceSymbol)) { + isSilence = 1; // true + } else { + isSilence = 0; // false + } + } + return isSilence == 1; + } + + public Allophone getAllophone() { + if (maryxmlElement != null) { + AllophoneSet allophoneSet = null; + Element voiceElement = (Element) MaryDomUtils.getAncestor(maryxmlElement, MaryXML.VOICE); + if (voiceElement != null) { + Voice v = Voice.getVoice(voiceElement); + if (v != null) { + allophoneSet = v.getAllophoneSet(); + } + } + if (allophoneSet == null) { + try { + allophoneSet = MaryRuntimeUtils.determineAllophoneSet(maryxmlElement); + } catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + String sampa; + if (maryxmlElement.getNodeName().equals(MaryXML.PHONE)) { + sampa = maryxmlElement.getAttribute("p"); + } else { + assert maryxmlElement.getNodeName().equals(MaryXML.BOUNDARY); + sampa = "_"; + } + return allophoneSet.getAllophone(sampa); + } + return null; + } + + public String toString() { + return name; + } +} diff --git a/marytts-unitselection/src/main/java/marytts/features/TargetFeatureComputer.java b/marytts-unitselection/src/main/java/marytts/features/TargetFeatureComputer.java new file mode 100644 index 0000000000..078ab325e5 --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/features/TargetFeatureComputer.java @@ -0,0 +1,348 @@ +/** + * Copyright 2008 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.features; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.StringReader; +import java.util.ArrayList; +import java.util.List; +import java.util.StringTokenizer; + + +/** + * Compute a given set of features for a Target. + * + * @author schroed + * + */ +public class TargetFeatureComputer { + protected ByteValuedFeatureProcessor[] byteValuedDiscreteFeatureProcessors; + protected ShortValuedFeatureProcessor[] shortValuedDiscreteFeatureProcessors; + protected ContinuousFeatureProcessor[] continuousFeatureProcessors; + + protected String pauseSymbol = null; + + protected FeatureDefinition featureDefinition = null; + + /** + * Construct a TargetFeatureComputer that knows how to compute features for a Target using the given set of feature processor + * names. These names must be known to the given Feature processor manager. + * + * @param manager + * @param featureProcessorNames + * a String containing the names of the feature processors to use, separated by white space, and in the right order + * (byte-valued discrete feature processors first, then short-valued, then continuous) + */ + public TargetFeatureComputer(FeatureProcessorManager manager, String featureProcessorNames) { + List byteValuedFeatureProcessors = new ArrayList(); + List shortValuedFeatureProcessors = new ArrayList(); + List continuousValuedFeatureProcessors = new ArrayList(); + + StringTokenizer st = new StringTokenizer(featureProcessorNames); + while (st.hasMoreTokens()) { + String name = st.nextToken(); + MaryFeatureProcessor fp = manager.getFeatureProcessor(name); + if (fp == null) { + throw new IllegalArgumentException("Unknown feature processor: " + name); + } else if (fp instanceof ByteValuedFeatureProcessor) { + byteValuedFeatureProcessors.add(fp); + } else if (fp instanceof ShortValuedFeatureProcessor) { + shortValuedFeatureProcessors.add(fp); + } else if (fp instanceof ContinuousFeatureProcessor) { + continuousValuedFeatureProcessors.add(fp); + } else { + throw new IllegalArgumentException("Unknown feature processor type " + fp.getClass() + " for feature processor: " + + name); + } + } + this.byteValuedDiscreteFeatureProcessors = (ByteValuedFeatureProcessor[]) byteValuedFeatureProcessors + .toArray(new ByteValuedFeatureProcessor[0]); + this.shortValuedDiscreteFeatureProcessors = (ShortValuedFeatureProcessor[]) shortValuedFeatureProcessors + .toArray(new ShortValuedFeatureProcessor[0]); + this.continuousFeatureProcessors = (ContinuousFeatureProcessor[]) continuousValuedFeatureProcessors + .toArray(new ContinuousFeatureProcessor[0]); + } + + /** + * Provide the feature definition that can be used to interpret the feature processors generated by this + * TargetFeatureComputer. + * + * @return + */ + public FeatureDefinition getFeatureDefinition() { + if (featureDefinition == null) { + StringBuilder sb = new StringBuilder(); + sb.append(FeatureDefinition.BYTEFEATURES).append("\n"); + for (int i = 0; i < byteValuedDiscreteFeatureProcessors.length; i++) { + sb.append(byteValuedDiscreteFeatureProcessors[i].getName()); + String[] values = byteValuedDiscreteFeatureProcessors[i].getValues(); + for (String v : values) { + sb.append(" ").append(v); + } + sb.append("\n"); + } + sb.append(FeatureDefinition.SHORTFEATURES).append("\n"); + for (int i = 0; i < shortValuedDiscreteFeatureProcessors.length; i++) { + sb.append(shortValuedDiscreteFeatureProcessors[i].getName()); + String[] values = shortValuedDiscreteFeatureProcessors[i].getValues(); + for (String v : values) { + sb.append(" ").append(v); + } + sb.append("\n"); + } + sb.append(FeatureDefinition.CONTINUOUSFEATURES).append("\n"); + for (int i = 0; i < continuousFeatureProcessors.length; i++) { + sb.append(continuousFeatureProcessors[i].getName()).append("\n"); + } + BufferedReader reader = new BufferedReader(new StringReader(sb.toString())); + try { + featureDefinition = new FeatureDefinition(reader, false); + } catch (IOException e) { + throw new RuntimeException("Problem creating feature definition", e); + } + } + return featureDefinition; + } + + /** + * Using the set of feature processors defined when creating the target feature computer, compute a feature vector for the + * target + * + * @param target + * @return a feature vector for the target + */ + public FeatureVector computeFeatureVector(Target target) { + byte[] byteFeatures = new byte[byteValuedDiscreteFeatureProcessors.length]; + short[] shortFeatures = new short[shortValuedDiscreteFeatureProcessors.length]; + float[] floatFeatures = new float[continuousFeatureProcessors.length]; + for (int i = 0; i < byteValuedDiscreteFeatureProcessors.length; i++) { + byteFeatures[i] = byteValuedDiscreteFeatureProcessors[i].process(target); + } + for (int i = 0; i < shortValuedDiscreteFeatureProcessors.length; i++) { + shortFeatures[i] = shortValuedDiscreteFeatureProcessors[i].process(target); + } + for (int i = 0; i < continuousFeatureProcessors.length; i++) { + floatFeatures[i] = continuousFeatureProcessors[i].process(target); + } + return new FeatureVector(byteFeatures, shortFeatures, floatFeatures, 0); + } + + /** + * For the given feature vector, convert each encoded value into its string representation. + * + * @param features + * a feature vector, which must match the feature processors known to this feature computer. + * @return a string in which the string values of all features are separated by spaces. + * @throws IllegalArgumentException + * if the number of byte-valued, short-valued or continuous elements in features do not match the set of feature + * processors in this feature computer. + */ + public String toStringValues(FeatureVector features) { + StringBuilder buf = new StringBuilder(); + byte[] bytes = features.getByteValuedDiscreteFeatures(); + short[] shorts = features.getShortValuedDiscreteFeatures(); + float[] floats = features.getContinuousFeatures(); + if (bytes.length != byteValuedDiscreteFeatureProcessors.length + || shorts.length != shortValuedDiscreteFeatureProcessors.length + || floats.length != continuousFeatureProcessors.length) { + throw new IllegalArgumentException("Number of features in argument does not match number of feature processors"); + } + for (int i = 0; i < bytes.length; i++) { + if (buf.length() > 0) + buf.append(" "); + buf.append(byteValuedDiscreteFeatureProcessors[i].getValues()[(int) bytes[i] & 0xff]); + } + for (int i = 0; i < shorts.length; i++) { + if (buf.length() > 0) + buf.append(" "); + buf.append(shortValuedDiscreteFeatureProcessors[i].getValues()[(int) shorts[i]]); + } + for (int i = 0; i < floats.length; i++) { + if (buf.length() > 0) + buf.append(" "); + buf.append(floats[i]); + } + return buf.toString(); + } + + public ByteValuedFeatureProcessor[] getByteValuedFeatureProcessors() { + return byteValuedDiscreteFeatureProcessors; + } + + public ShortValuedFeatureProcessor[] getShortValuedFeatureProcessors() { + return shortValuedDiscreteFeatureProcessors; + } + + public ContinuousFeatureProcessor[] getContinuousFeatureProcessors() { + return continuousFeatureProcessors; + } + + /** + * List the names of all feature processors. The first line starts with "ByteValuedFeatureProcessors", followed by the list of + * names of the byte-valued feature processors; the second line starts with "ShortValuedFeatureProcessors", followed by the + * list of names of the short-valued feature processors; and the third line starts with "ContinuousFeatureProcessors", + * followed by the list of names of the continuous feature processors. + * + * @return a string with the names. + */ + public String getAllFeatureProcessorNames() { + return "ByteValuedFeatureProcessors " + getByteValuedFeatureProcessorNames() + "\n" + "ShortValuedFeatureProcessors " + + getShortValuedFeatureProcessorNames() + "\n" + "ContinuousFeatureProcessors " + + getContinuousFeatureProcessorNames() + "\n"; + } + + /** + * List the names of all byte-valued feature processors, separated by space characters. + * + * @return a string with the names. + */ + public String getByteValuedFeatureProcessorNames() { + StringBuilder buf = new StringBuilder(); + for (int i = 0; i < byteValuedDiscreteFeatureProcessors.length; i++) { + if (i > 0) + buf.append(" "); + buf.append(byteValuedDiscreteFeatureProcessors[i].getName()); + } + return buf.toString(); + } + + /** + * List the names of all short-valued feature processors, separated by space characters. + * + * @return a string with the names. + */ + public String getShortValuedFeatureProcessorNames() { + StringBuilder buf = new StringBuilder(); + for (int i = 0; i < shortValuedDiscreteFeatureProcessors.length; i++) { + if (i > 0) + buf.append(" "); + buf.append(shortValuedDiscreteFeatureProcessors[i].getName()); + } + return buf.toString(); + } + + /** + * List the names of all byte-valued feature processors, separated by space characters. + * + * @return a string with the names. + */ + public String getContinuousFeatureProcessorNames() { + StringBuilder buf = new StringBuilder(); + for (int i = 0; i < continuousFeatureProcessors.length; i++) { + if (i > 0) + buf.append(" "); + buf.append(continuousFeatureProcessors[i].getName()); + } + return buf.toString(); + } + + /** + * List the names and values of all feature processors. The section describing the byte-valued feature processors starts with + * the string "ByteValuedFeatureProcessors" in a line by itself, followed by the list of names and values of the byte-valued + * feature processors, as described in getByteValuedFeatureProcessorNamesAndValues(). The section describing the short-valued + * feature processors starts with the string "ShortValuedFeatureProcessors" in a line by itself, followed by the list of names + * and values of the short-valued feature processors, as described in getShortValuedFeatureProcessorNamesAndValues(). The + * section describing the continuous feature processors starts with the string "ContinuousFeatureProcessors" in a line by + * itself, followed by the list of names and values of the continuous feature processors, as described in + * getContinuousFeatureProcessorNamesAndValues(). + * + * @return a string with the names and values. + */ + public String getAllFeatureProcessorNamesAndValues() { + return FeatureDefinition.BYTEFEATURES + "\n" + getByteValuedFeatureProcessorNamesAndValues() + + FeatureDefinition.SHORTFEATURES + "\n" + getShortValuedFeatureProcessorNamesAndValues() + + FeatureDefinition.CONTINUOUSFEATURES + "\n" + getContinuousFeatureProcessorNamesAndValues(); + } + + /** + * List the names of all byte-valued feature processors and their possible values. Each line starts with the name of a feature + * processor, followed by the full list of string values, separated by space characters. The values are ordered so that their + * position corresponds to the byte value. + * + * @return a string with the names and values. + */ + public String getByteValuedFeatureProcessorNamesAndValues() { + StringBuilder buf = new StringBuilder(); + for (int i = 0; i < byteValuedDiscreteFeatureProcessors.length; i++) { + buf.append(byteValuedDiscreteFeatureProcessors[i].getName()); + String[] values = byteValuedDiscreteFeatureProcessors[i].getValues(); + for (int j = 0; j < values.length; j++) { + buf.append(" "); + buf.append(values[j]); + } + buf.append("\n"); + } + return buf.toString(); + } + + /** + * List the names of all short-valued feature processors and their possible values. Each line starts with the name of a + * feature processor, followed by the full list of string values, separated by space characters. The values are ordered so + * that their position corresponds to the short value. + * + * @return a string with the names and values. + */ + public String getShortValuedFeatureProcessorNamesAndValues() { + StringBuilder buf = new StringBuilder(); + for (int i = 0; i < shortValuedDiscreteFeatureProcessors.length; i++) { + buf.append(shortValuedDiscreteFeatureProcessors[i].getName()); + String[] values = shortValuedDiscreteFeatureProcessors[i].getValues(); + for (int j = 0; j < values.length; j++) { + buf.append(" "); + buf.append(values[j]); + } + buf.append("\n"); + } + return buf.toString(); + } + + /** + * List the names of all continuous feature processors and their possible values. Each line starts with the name of a feature + * processor, followed by the value "float" to indicate that the list of possible values is not limited. + * + * @return a string with the names and values. + */ + public String getContinuousFeatureProcessorNamesAndValues() { + StringBuilder buf = new StringBuilder(); + for (int i = 0; i < continuousFeatureProcessors.length; i++) { + buf.append(continuousFeatureProcessors[i].getName()); + buf.append(" float\n"); + } + return buf.toString(); + } + + /** + * Get the pause symbol as associated with the "phone" feature processor used. + * + * @return + */ + public String getPauseSymbol() { + if (pauseSymbol == null) { + for (MaryFeatureProcessor fp : byteValuedDiscreteFeatureProcessors) { + if (fp instanceof MaryLanguageFeatureProcessors.Phone) { + pauseSymbol = ((MaryLanguageFeatureProcessors.Phone) fp).getPauseSymbol(); + break; + } + } + } + return pauseSymbol; + } +} diff --git a/marytts-unitselection/src/main/java/marytts/unitselection/UnitSelectionSynthesizer.java b/marytts-unitselection/src/main/java/marytts/unitselection/UnitSelectionSynthesizer.java new file mode 100644 index 0000000000..69ce8c53df --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/unitselection/UnitSelectionSynthesizer.java @@ -0,0 +1,270 @@ +/** + * Copyright 2006 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.unitselection; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringReader; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Iterator; +import java.util.List; +import java.util.Locale; +import java.util.StringTokenizer; + +import javax.sound.sampled.AudioFormat; +import javax.sound.sampled.AudioInputStream; + +import marytts.datatypes.MaryData; +import marytts.datatypes.MaryDataType; +import marytts.datatypes.MaryXML; +import marytts.exceptions.SynthesisException; +import marytts.modules.synthesis.Voice; +import marytts.modules.synthesis.WaveformSynthesizer; +import marytts.modules.synthesis.Voice.Gender; +import marytts.server.MaryProperties; +import marytts.unitselection.concat.UnitConcatenator; +import marytts.unitselection.concat.BaseUnitConcatenator.UnitData; +import marytts.unitselection.data.Unit; +import marytts.unitselection.data.UnitDatabase; +import marytts.features.HalfPhoneTarget; +import marytts.unitselection.select.SelectedUnit; +import marytts.features.Target; +import marytts.unitselection.select.UnitSelector; +import marytts.util.MaryUtils; +import marytts.util.dom.MaryNormalisedWriter; +import marytts.util.dom.NameNodeFilter; + +import org.apache.log4j.Level; +import org.apache.log4j.Logger; +import org.w3c.dom.Element; +import org.w3c.dom.traversal.DocumentTraversal; +import org.w3c.dom.traversal.NodeFilter; +import org.w3c.dom.traversal.TreeWalker; + +/** + * Builds and synthesizes unit selection voices + * + * @author Marc Schröder, Anna Hunecke + * + */ + +public class UnitSelectionSynthesizer implements WaveformSynthesizer { + /** + * A map with Voice objects as keys, and Lists of UtteranceProcessors as values. Idea: For a given voice, find the list of + * utterance processors to apply. + */ + private Logger logger; + + public UnitSelectionSynthesizer() { + } + + /** + * Start up the waveform synthesizer. This must be called once before calling synthesize(). + */ + public void startup() throws Exception { + logger = MaryUtils.getLogger("UnitSelectionSynthesizer"); + // Register UnitSelection voices: + logger.debug("Register UnitSelection voices:"); + List voiceNames = MaryProperties.getList("unitselection.voices.list"); + for (String voiceName : voiceNames) { + long time = System.currentTimeMillis(); + Voice unitSelVoice = new UnitSelectionVoice(voiceName, this); + logger.debug("Voice '" + unitSelVoice + "'"); + Voice.registerVoice(unitSelVoice); + long newtime = System.currentTimeMillis() - time; + logger.info("Loading of voice " + voiceName + " took " + newtime + " milliseconds"); + } + logger.info("started."); + } + + /** + * Perform a power-on self test by processing some example input data. + * + * @throws Error + * if the module does not work properly. + */ + public void powerOnSelfTest() throws Error { + try { + Collection myVoices = Voice.getAvailableVoices(this); + if (myVoices.size() == 0) { + return; + } + UnitSelectionVoice unitSelVoice = (UnitSelectionVoice) myVoices.iterator().next(); + assert unitSelVoice != null; + MaryData in = new MaryData(MaryDataType.get("ACOUSTPARAMS"), unitSelVoice.getLocale()); + if (!unitSelVoice.getDomain().equals("general")) { + logger.info("Cannot perform power-on self test using limited-domain voice '" + unitSelVoice.getName() + + "' - skipping."); + return; + } + String exampleText = MaryDataType.ACOUSTPARAMS.exampleText(unitSelVoice.getLocale()); + if (exampleText != null) { + in.readFrom(new StringReader(exampleText)); + in.setDefaultVoice(unitSelVoice); + if (in == null) { + System.out.println(exampleText + " is null"); + } + List tokensAndBoundaries = new ArrayList(); + TreeWalker tw = ((DocumentTraversal) in.getDocument()).createTreeWalker(in.getDocument(), + NodeFilter.SHOW_ELEMENT, new NameNodeFilter(new String[] { MaryXML.TOKEN, MaryXML.BOUNDARY }), false); + Element el = null; + while ((el = (Element) tw.nextNode()) != null) + tokensAndBoundaries.add(el); + AudioInputStream ais = synthesize(tokensAndBoundaries, unitSelVoice, null); + assert ais != null; + } else { + logger.debug("No example text -- no power-on self test!"); + } + } catch (Throwable t) { + t.printStackTrace(); + throw new Error("Module " + toString() + ": Power-on self test failed.", t); + } + logger.info("Power-on self test complete."); + } + + /** + * {@inheritDoc} + */ + public AudioInputStream synthesize(List tokensAndBoundaries, Voice voice, String outputParams) + throws SynthesisException { + assert voice instanceof UnitSelectionVoice; + UnitSelectionVoice v = (UnitSelectionVoice) voice; + UnitDatabase udb = v.getDatabase(); + // Select: + UnitSelector unitSel = v.getUnitSelector(); + UnitConcatenator unitConcatenator; + if (outputParams != null && outputParams.contains("MODIFICATION")) { + unitConcatenator = v.getModificationConcatenator(); + } else { + unitConcatenator = v.getConcatenator(); + } + // TODO: check if we actually need to access v.getDatabase() here + UnitDatabase database = v.getDatabase(); + logger.debug("Selecting units with a " + unitSel.getClass().getName() + " from a " + database.getClass().getName()); + List selectedUnits = unitSel.selectUnits(tokensAndBoundaries, voice); + // if (logger.getEffectiveLevel().equals(Level.DEBUG)) { + // StringWriter sw = new StringWriter(); + // PrintWriter pw = new PrintWriter(sw); + // for (Iterator selIt=selectedUnits.iterator(); selIt.hasNext(); ) + // pw.println(selIt.next()); + // logger.debug("Units selected:\n"+sw.toString()); + // } + + // Concatenate: + logger.debug("Now creating audio with a " + unitConcatenator.getClass().getName()); + AudioInputStream audio = null; + try { + audio = unitConcatenator.getAudio(selectedUnits); + } catch (IOException ioe) { + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw); + for (Iterator selIt = selectedUnits.iterator(); selIt.hasNext();) + pw.println(selIt.next()); + throw new SynthesisException("Problems generating audio for unit chain: " + sw.toString(), ioe); + } + + // Propagate unit durations to XML tree: + float endInSeconds = 0; + float durLeftHalfInSeconds = 0; + String unitString = ""; + String unitAttrName = "units"; // name of the attribute that is added for unit selection diagnostics + for (SelectedUnit su : selectedUnits) { + Target t = su.getTarget(); + boolean halfphone = (t instanceof HalfPhoneTarget); + Object concatenationData = su.getConcatenationData(); + assert concatenationData instanceof UnitData; + UnitData unitData = (UnitData) concatenationData; + Unit unit = su.getUnit(); + + // For the unit durations, keep record in floats because of precision; + // convert to millis only at export time, and re-compute duration in millis + // from the end in millis, to avoid discrepancies due to rounding + int unitDurationInSamples = unitData.getUnitDuration(); + float unitDurationInSeconds = unitDurationInSamples / (float) database.getUnitFileReader().getSampleRate(); + int prevEndInMillis = (int) (1000 * endInSeconds); + endInSeconds += unitDurationInSeconds; + int endInMillis = (int) (1000 * endInSeconds); + int unitDurationInMillis = endInMillis - prevEndInMillis; + unitString = t.getName() + " " + udb.getFilename(unit) + " " + unit.index + " " + unitDurationInSeconds; + if (halfphone) { + if (((HalfPhoneTarget) t).isLeftHalf()) { + durLeftHalfInSeconds = unitDurationInSeconds; + } else { // right half + // re-compute unit duration from both halves + float totalUnitDurInSeconds = durLeftHalfInSeconds + unitDurationInSeconds; + float prevEndInSeconds = endInSeconds - totalUnitDurInSeconds; + prevEndInMillis = (int) (1000 * prevEndInSeconds); + unitDurationInMillis = endInMillis - prevEndInMillis; + durLeftHalfInSeconds = 0; + } + } + + Element maryxmlElement = t.getMaryxmlElement(); + if (maryxmlElement != null) { + if (maryxmlElement.getNodeName().equals(MaryXML.PHONE)) { + if (!maryxmlElement.hasAttribute("d") || !maryxmlElement.hasAttribute("end")) { + throw new IllegalStateException("No duration information in MaryXML -- check log file" + + " for messages warning about unloadable acoustic models" + + " instead of voice-specific acoustic feature predictors"); + } + // int oldD = Integer.parseInt(maryxmlElement.getAttribute("d")); + // int oldEnd = Integer.parseInt(maryxmlElement.getAttribute("end")); + // double doubleEnd = Double.parseDouble(maryxmlElement.getAttribute("end")); + // int oldEnd = (int)(doubleEnd * 1000); + maryxmlElement.setAttribute("d", String.valueOf(unitDurationInMillis)); + maryxmlElement.setAttribute("end", String.valueOf(endInSeconds)); + // the following messes up all end values! + // if (oldEnd == oldD) { + // // start new end computation + // endInSeconds = unitDurationInSeconds; + // } + } else { // not a PHONE + assert maryxmlElement.getNodeName().equals(MaryXML.BOUNDARY); + maryxmlElement.setAttribute("duration", String.valueOf(unitDurationInMillis)); + } + if (maryxmlElement.hasAttribute(unitAttrName)) { + String prevUnitString = maryxmlElement.getAttribute(unitAttrName); + maryxmlElement.setAttribute(unitAttrName, prevUnitString + "; " + unitString); + } else { + maryxmlElement.setAttribute(unitAttrName, unitString); + } + } else { + logger.debug("Unit " + su.getTarget().getName() + " of length " + unitDurationInMillis + + " ms has no maryxml element."); + } + } + if (logger.getEffectiveLevel().equals(Level.DEBUG)) { + try { + MaryNormalisedWriter writer = new MaryNormalisedWriter(); + ByteArrayOutputStream debugOut = new ByteArrayOutputStream(); + writer.output(tokensAndBoundaries.get(0).getOwnerDocument(), debugOut); + logger.debug("Propagating the realised unit durations to the XML tree: \n" + debugOut.toString()); + } catch (Exception e) { + logger.warn("Problem writing XML to logfile: " + e); + } + } + + return audio; + } + +} diff --git a/marytts-unitselection/src/main/java/marytts/unitselection/UnitSelectionVoice.java b/marytts-unitselection/src/main/java/marytts/unitselection/UnitSelectionVoice.java new file mode 100644 index 0000000000..c11d484424 --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/unitselection/UnitSelectionVoice.java @@ -0,0 +1,325 @@ +/** + * Portions Copyright 2006 DFKI GmbH. + * Portions Copyright 2001 Sun Microsystems, Inc. + * Portions Copyright 1999-2001 Language Technologies Institute, + * Carnegie Mellon University. + * All Rights Reserved. Use is subject to license terms. + * + * Permission is hereby granted, free of charge, to use and distribute + * this software and its documentation without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of this work, and to + * permit persons to whom this work is furnished to do so, subject to + * the following conditions: + * + * 1. The code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * 2. Any modifications must be clearly marked as such. + * 3. Original authors' names are not deleted. + * 4. The authors' names are not used to endorse or promote products + * derived from this software without specific prior written + * permission. + * + * DFKI GMBH AND THE CONTRIBUTORS TO THIS WORK DISCLAIM ALL WARRANTIES WITH + * REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL DFKI GMBH NOR THE + * CONTRIBUTORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL + * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR + * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS + * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF + * THIS SOFTWARE. + */ +package marytts.unitselection; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.lang.reflect.Constructor; +import java.util.Locale; + +import javax.sound.sampled.AudioFormat; + +import marytts.cart.CART; +import marytts.cart.io.MaryCARTReader; +import marytts.exceptions.MaryConfigurationException; +import marytts.features.FeatureDefinition; +import marytts.features.FeatureProcessorManager; +import marytts.features.FeatureRegistry; +import marytts.modules.synthesis.Voice; +import marytts.modules.synthesis.WaveformSynthesizer; +import marytts.server.MaryProperties; +import marytts.unitselection.concat.FdpsolaUnitConcatenator; +import marytts.unitselection.concat.UnitConcatenator; +import marytts.unitselection.data.TimelineReader; +import marytts.unitselection.data.UnitDatabase; +import marytts.unitselection.data.UnitFileReader; +import marytts.unitselection.select.JoinCostFunction; +import marytts.unitselection.select.JoinModelCost; +import marytts.unitselection.select.StatisticalCostFunction; +import marytts.unitselection.select.TargetCostFunction; +import marytts.unitselection.select.UnitSelector; + +/** + * A Unit Selection Voice + * + */ +public class UnitSelectionVoice extends Voice { + + protected UnitDatabase database; + protected UnitSelector unitSelector; + protected UnitConcatenator concatenator; + protected UnitConcatenator modificationConcatenator; + protected String domain; + protected String name; + protected CART[] f0Carts; + protected String exampleText; + + public UnitSelectionVoice(String name, WaveformSynthesizer synthesizer) throws MaryConfigurationException { + super(name, synthesizer); + + try { + this.name = name; + String header = "voice." + name; + + domain = MaryProperties.needProperty(header + ".domain"); + InputStream exampleTextStream = null; + if (!domain.equals("general")) { // limited domain voices must have example text; + exampleTextStream = MaryProperties.needStream(header + ".exampleTextFile"); + } else { // general domain voices can have example text: + exampleTextStream = MaryProperties.getStream(header + ".exampleTextFile"); + } + if (exampleTextStream != null) { + readExampleText(exampleTextStream); + } + + FeatureProcessorManager featProcManager = FeatureRegistry.getFeatureProcessorManager(this); + if (featProcManager == null) + featProcManager = FeatureRegistry.getFeatureProcessorManager(getLocale()); + if (featProcManager == null) + throw new MaryConfigurationException("No feature processor manager for voice '" + name + "' (locale " + + getLocale() + ")"); + + // build and load targetCostFunction + logger.debug("...loading target cost function..."); + String featureFileName = MaryProperties.needFilename(header + ".featureFile"); + InputStream targetWeightStream = MaryProperties.getStream(header + ".targetCostWeights"); + String targetCostClass = MaryProperties.needProperty(header + ".targetCostClass"); + TargetCostFunction targetFunction = (TargetCostFunction) Class.forName(targetCostClass).newInstance(); + targetFunction.load(featureFileName, targetWeightStream, featProcManager); + + // build joinCostFunction + logger.debug("...loading join cost function..."); + String joinCostClass = MaryProperties.needProperty(header + ".joinCostClass"); + JoinCostFunction joinFunction = (JoinCostFunction) Class.forName(joinCostClass).newInstance(); + if (joinFunction instanceof JoinModelCost) { + ((JoinModelCost) joinFunction).setFeatureDefinition(targetFunction.getFeatureDefinition()); + } + joinFunction.init(header); + + // build sCost function + StatisticalCostFunction sCostFunction = null; + boolean useSCost = MaryProperties.getBoolean(header + ".useSCost", false); + if (useSCost) { + logger.debug("...loading scost function..."); + String sCostClass = MaryProperties.needProperty(header + ".sCostClass"); + sCostFunction = (StatisticalCostFunction) Class.forName(sCostClass).newInstance(); + sCostFunction.init(header); + } + + // Build the various file readers + logger.debug("...loading units file..."); + String unitReaderClass = MaryProperties.needProperty(header + ".unitReaderClass"); + String unitsFile = MaryProperties.needFilename(header + ".unitsFile"); + UnitFileReader unitReader = (UnitFileReader) Class.forName(unitReaderClass).newInstance(); + unitReader.load(unitsFile); + + logger.debug("...loading cart file..."); + // String cartReaderClass = MaryProperties.needProperty(header+".cartReaderClass"); + InputStream cartStream = MaryProperties.needStream(header + ".cartFile"); + CART cart = new MaryCARTReader().loadFromStream(cartStream); + cartStream.close(); + // get the backtrace information + int backtrace = MaryProperties.getInteger(header + ".cart.backtrace", 100); + + logger.debug("...loading audio time line..."); + String timelineReaderClass = MaryProperties.needProperty(header + ".audioTimelineReaderClass"); + String timelineFile = MaryProperties.needFilename(header + ".audioTimelineFile"); + Class theClass = Class.forName(timelineReaderClass).asSubclass(TimelineReader.class); + // Now invoke Constructor with one String argument + Class[] constructorArgTypes = new Class[] { String.class }; + Object[] args = new Object[] { timelineFile }; + Constructor constructor = (Constructor) theClass + .getConstructor(constructorArgTypes); + TimelineReader timelineReader = constructor.newInstance(args); + + // optionally, get basename timeline + String basenameTimelineFile = MaryProperties.getFilename(header + ".basenameTimeline"); + TimelineReader basenameTimelineReader = null; + if (basenameTimelineFile != null) { + logger.debug("...loading basename time line..."); + basenameTimelineReader = new TimelineReader(basenameTimelineFile); + } + + // build and load database + logger.debug("...instantiating database..."); + String databaseClass = MaryProperties.needProperty(header + ".databaseClass"); + database = (UnitDatabase) Class.forName(databaseClass).newInstance(); + if (useSCost) { + database.load(targetFunction, joinFunction, sCostFunction, unitReader, cart, timelineReader, + basenameTimelineReader, backtrace); + } else { + database.load(targetFunction, joinFunction, unitReader, cart, timelineReader, basenameTimelineReader, backtrace); + } + + // build Selector + logger.debug("...instantiating unit selector..."); + String selectorClass = MaryProperties.needProperty(header + ".selectorClass"); + unitSelector = (UnitSelector) Class.forName(selectorClass).newInstance(); + float targetCostWeights = Float.parseFloat(MaryProperties.getProperty(header + ".viterbi.wTargetCosts", "0.33")); + int beamSize = MaryProperties.getInteger(header + ".viterbi.beamsize", 100); + if (!useSCost) { + unitSelector.load(database, targetCostWeights, beamSize); + } else { + float sCostWeights = Float.parseFloat(MaryProperties.getProperty(header + ".viterbi.wSCosts", "0.33")); + unitSelector.load(database, targetCostWeights, sCostWeights, beamSize); + } + + // samplingRate -> bin, audioformat -> concatenator + // build Concatenator + logger.debug("...instantiating unit concatenator..."); + String concatenatorClass = MaryProperties.needProperty(header + ".concatenatorClass"); + concatenator = (UnitConcatenator) Class.forName(concatenatorClass).newInstance(); + concatenator.load(database); + + // TODO: this can be deleted at the same time as CARTF0Modeller + // see if there are any voice-specific duration and f0 models to load + f0Carts = null; + InputStream leftF0CartStream = MaryProperties.getStream(header + ".f0.cart.left"); + if (leftF0CartStream != null) { + logger.debug("...loading f0 trees..."); + f0Carts = new CART[3]; + f0Carts[0] = new MaryCARTReader().loadFromStream(leftF0CartStream); + leftF0CartStream.close(); + // mid cart: + InputStream midF0CartStream = MaryProperties.needStream(header + ".f0.cart.mid"); + f0Carts[1] = new MaryCARTReader().loadFromStream(midF0CartStream); + midF0CartStream.close(); + // right cart: + InputStream rightF0CartStream = MaryProperties.needStream(header + ".f0.cart.right"); + f0Carts[2] = new MaryCARTReader().loadFromStream(rightF0CartStream); + rightF0CartStream.close(); + } + } catch (MaryConfigurationException mce) { + throw mce; + } catch (Exception ex) { + throw new MaryConfigurationException("Cannot build unit selection voice '" + name + "'", ex); + } + + } + + /** + * Gets the database of this voice + * + * @return the database + */ + public UnitDatabase getDatabase() { + return database; + } + + /** + * Gets the unit selector of this voice + * + * @return the unit selector + */ + public UnitSelector getUnitSelector() { + return unitSelector; + } + + /** + * Gets the unit concatenator of this voice + * + * @return the unit selector + */ + public UnitConcatenator getConcatenator() { + return concatenator; + } + + /** + * Get the modification UnitConcatenator of this voice + * + * @return the modifying UnitConcatenator + */ + public UnitConcatenator getModificationConcatenator() { + if (modificationConcatenator == null) { + // get sensible minimum and maximum values: + try { + // initialize with values from properties: + double minTimeScaleFactor = Double.parseDouble(MaryProperties.getProperty("voice." + name + + ".prosody.modification.duration.factor.minimum")); + double maxTimeScaleFactor = Double.parseDouble(MaryProperties.getProperty("voice." + name + + ".prosody.modification.duration.factor.maximum")); + double minPitchScaleFactor = Double.parseDouble(MaryProperties.getProperty("voice." + name + + ".prosody.modification.f0.factor.minimum")); + double maxPitchScaleFactor = Double.parseDouble(MaryProperties.getProperty("voice." + name + + ".prosody.modification.f0.factor.maximum")); + logger.debug("Initializing FD-PSOLA unit concatenator with the following parameter thresholds:"); + logger.debug("minimum duration modification factor: " + minTimeScaleFactor); + logger.debug("maximum duration modification factor: " + maxTimeScaleFactor); + logger.debug("minimum F0 modification factor: " + minPitchScaleFactor); + logger.debug("maximum F0 modification factor: " + maxPitchScaleFactor); + modificationConcatenator = new FdpsolaUnitConcatenator(minTimeScaleFactor, maxTimeScaleFactor, + minPitchScaleFactor, maxPitchScaleFactor); + } catch (Exception e) { + // ignore -- defaults will be used + logger.debug("Initializing FD-PSOLA unit concatenator with default parameter thresholds."); + modificationConcatenator = new FdpsolaUnitConcatenator(); + } + modificationConcatenator.load(database); + } + return modificationConcatenator; + } + + /** + * Gets the domain of this voice + * + * @return the domain + */ + public String getDomain() { + return domain; + } + + public String getExampleText() { + if (exampleText == null) { + return ""; + } else { + return exampleText; + } + } + + public void readExampleText(InputStream in) throws IOException { + BufferedReader reader = new BufferedReader(new InputStreamReader(in, "UTF-8")); + StringBuilder sb = new StringBuilder(); + String line = reader.readLine(); + while (line != null) { + if (!line.startsWith("***")) { + sb.append(line + "\n"); + } + line = reader.readLine(); + } + exampleText = sb.toString(); + } + + public CART[] getF0Trees() { + return f0Carts; + } + + public FeatureDefinition getF0CartsFeatDef() { + if (f0Carts == null || f0Carts.length < 1) + return null; + return f0Carts[0].getFeatureDefinition(); + } + +} diff --git a/marytts-unitselection/src/main/java/marytts/unitselection/analysis/HnmVoiceDataDumper.java b/marytts-unitselection/src/main/java/marytts/unitselection/analysis/HnmVoiceDataDumper.java new file mode 100644 index 0000000000..5e1d7142e4 --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/unitselection/analysis/HnmVoiceDataDumper.java @@ -0,0 +1,126 @@ +/** + * Copyright 2010 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ + +package marytts.unitselection.analysis; + +import java.io.IOException; + +import javax.sound.sampled.AudioFormat; + +import marytts.exceptions.MaryConfigurationException; +import marytts.signalproc.sinusoidal.hntm.analysis.HntmAnalyzerParams; +import marytts.signalproc.sinusoidal.hntm.analysis.HntmSpeechFrame; +import marytts.signalproc.sinusoidal.hntm.analysis.HntmSpeechSignal; +import marytts.signalproc.sinusoidal.hntm.synthesis.HntmSynthesizedSignal; +import marytts.signalproc.sinusoidal.hntm.synthesis.HntmSynthesizer; +import marytts.signalproc.sinusoidal.hntm.synthesis.HntmSynthesizerParams; +import marytts.unitselection.data.HnmDatagram; +import marytts.unitselection.data.HnmTimelineReader; +import marytts.util.data.BufferedDoubleDataSource; +import marytts.util.data.Datagram; +import marytts.util.data.audio.DDSAudioInputStream; +import marytts.util.math.MathUtils; + +/** + * Convenience class to dump relevant data from a HNM unit selection voice to a Praat TextGrid and a wav file for inspection of + * timeline data in external tools (e.g. Praat, WaveSurfer, etc.) + * + * @author steiner + * + */ +public class HnmVoiceDataDumper extends VoiceDataDumper { + + private AudioFormat audioformat; + + public HnmVoiceDataDumper() { + super(); + } + + /** + * {@inheritDoc} + *

+ * Also set the audioFormat needed in {@link #getSamples(Datagram[])} + */ + @Override + protected HnmTimelineReader loadAudioTimeline(String fileName) throws IOException, MaryConfigurationException { + HnmTimelineReader audioTimeline = new HnmTimelineReader(fileName); + int sampleRate = audioTimeline.getSampleRate(); + this.audioformat = new AudioFormat(AudioFormat.Encoding.PCM_SIGNED, // encoding + sampleRate, // samples per second + 16, // bits per sample + 1, // mono + 2, // nr. of bytes per frame + sampleRate, // nr. of frames per second + true); // big-endian; + return audioTimeline; + } + + /** + * {@inheritDoc} + *

+ * For {@link HnmDatagram}s, the samples must be resynthesized from the HntmSpeechFrame in each HnmDatagram. This requires + * quite a bit of processing. + */ + @Override + protected byte[] getSamples(Datagram[] datagrams) throws IOException { + // init required objects: + HntmSynthesizer hnmSynthesizer = new HntmSynthesizer(); + HntmAnalyzerParams hnmAnalysisParams = new HntmAnalyzerParams(); + HntmSynthesizerParams hnmSynthesisParams = new HntmSynthesizerParams(); + + // get duration from datagrams: + float originalDurationInSeconds = 0; + for (Datagram datagram : datagrams) { + HnmDatagram hnmDatagram = (HnmDatagram) datagram; + originalDurationInSeconds += hnmDatagram.getFrame().deltaAnalysisTimeInSeconds; + } + + // generate HNM signal from frames, correcting the analysis times: + HntmSpeechSignal hnmSpeechSignal = new HntmSpeechSignal(datagrams.length, unitDB.getAudioTimeline().getSampleRate(), + originalDurationInSeconds); + float tAnalysisInSeconds = 0; + for (int i = 0; i < datagrams.length; i++) { + HntmSpeechFrame hnmSpeechFrame = ((HnmDatagram) datagrams[i]).getFrame(); + // correct analysis time: + tAnalysisInSeconds += hnmSpeechFrame.deltaAnalysisTimeInSeconds; + hnmSpeechFrame.tAnalysisInSeconds = tAnalysisInSeconds; + hnmSpeechSignal.frames[i] = hnmSpeechFrame; + } + + // synthesize signal + HntmSynthesizedSignal hnmSynthesizedSignal = hnmSynthesizer.synthesize(hnmSpeechSignal, null, null, null, null, + hnmAnalysisParams, hnmSynthesisParams); + + // scale amplitude: + double[] output = MathUtils.multiply(hnmSynthesizedSignal.output, 1.0 / 32768.0); + + // repack output into byte array: + BufferedDoubleDataSource buffer = new BufferedDoubleDataSource(output); + DDSAudioInputStream audio = new DDSAudioInputStream(buffer, audioformat); + byte[] samples = new byte[(int) audio.getFrameLength() * audioformat.getFrameSize()]; + audio.read(samples); + return samples; + } + + public static void main(String[] args) throws Exception { + new HnmVoiceDataDumper().dumpData(args[0]); + } + +} diff --git a/marytts-unitselection/src/main/java/marytts/unitselection/analysis/Phone.java b/marytts-unitselection/src/main/java/marytts/unitselection/analysis/Phone.java new file mode 100644 index 0000000000..8b0f5cb4a0 --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/unitselection/analysis/Phone.java @@ -0,0 +1,748 @@ +/** + * Copyright 2010 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ + +package marytts.unitselection.analysis; + +import java.util.Arrays; + +import marytts.modules.phonemiser.Allophone; +import marytts.unitselection.concat.BaseUnitConcatenator.UnitData; +import marytts.features.HalfPhoneTarget; +import marytts.unitselection.select.SelectedUnit; +import marytts.util.data.Datagram; +import marytts.util.math.MathUtils; + +import org.apache.commons.lang.ArrayUtils; +import org.w3c.dom.Element; + +/** + * Convenience class containing the selected units and targets of a phone segment, and a host of getters to access their prosodic + * attributes + * + * @author steiner + * + */ +public class Phone { + private HalfPhoneTarget leftTarget; + + private HalfPhoneTarget rightTarget; + + private SelectedUnit leftUnit; + + private SelectedUnit rightUnit; + + private double sampleRate; + + private double[] leftF0Targets; + + private double[] rightF0Targets; + + /** + * Main constructor + * + * @param leftUnit + * which can be null + * @param rightUnit + * which can be null + * @param sampleRate + * of the TimelineReader containing the SelectedUnits, needed to provide duration information + * @throws IllegalArgumentException + * if both the left and right units are null + */ + public Phone(SelectedUnit leftUnit, SelectedUnit rightUnit, int sampleRate) throws IllegalArgumentException { + this.leftUnit = leftUnit; + this.rightUnit = rightUnit; + this.sampleRate = (double) sampleRate; + // targets are extracted from the units for easier access: + try { + this.leftTarget = (HalfPhoneTarget) leftUnit.getTarget(); + } catch (NullPointerException e) { + // leave at null + } + try { + this.rightTarget = (HalfPhoneTarget) rightUnit.getTarget(); + } catch (NullPointerException e) { + if (leftTarget == null) { + throw new IllegalArgumentException("A phone's left and right halves cannot both be null!"); + } else { + // leave at null + } + } + } + + /** + * Get the left selected halfphone unit of this phone + * + * @return the left unit, or null if there is no left unit + */ + public SelectedUnit getLeftUnit() { + return leftUnit; + } + + /** + * Get the right selected halfphone unit of this phone + * + * @return the right unit, or null if there is no right unit + */ + public SelectedUnit getRightUnit() { + return rightUnit; + } + + /** + * Get the left halfphone target of this phone + * + * @return the left target, or null if there is no left target + */ + public HalfPhoneTarget getLeftTarget() { + return leftTarget; + } + + /** + * Get the right halfphone target of this phone + * + * @return the right target, or null if there is no right target + */ + public HalfPhoneTarget getRightTarget() { + return rightTarget; + } + + /** + * Get the datagrams from a SelectedUnit + * + * @param unit + * the SelectedUnit + * @return the datagrams in an array, or null if unit is null + */ + private Datagram[] getUnitFrames(SelectedUnit unit) { + UnitData unitData = getUnitData(unit); + Datagram[] frames = null; + try { + frames = unitData.getFrames(); + } catch (NullPointerException e) { + // leave at null + } + return frames; + } + + /** + * Get this phone's left unit's Datagrams + * + * @return the left unit's Datagrams in an array, or null if there is no left unit + */ + public Datagram[] getLeftUnitFrames() { + return getUnitFrames(leftUnit); + } + + /** + * Get this phone's right unit's Datagrams + * + * @return the right unit's Datagrams in an array, or null if there is no right unit + */ + public Datagram[] getRightUnitFrames() { + return getUnitFrames(rightUnit); + } + + /** + * Get all Datagrams in this phone's units + * + * @return the left and right unit's Datagrams in an array + */ + public Datagram[] getUnitDataFrames() { + Datagram[] leftUnitFrames = getLeftUnitFrames(); + Datagram[] rightUnitFrames = getRightUnitFrames(); + Datagram[] frames = (Datagram[]) ArrayUtils.addAll(leftUnitFrames, rightUnitFrames); + return frames; + } + + /** + * Get the number of Datagrams in a SelectedUnit's UnitData + * + * @param unit + * whose Datagrams to count + * @return the number of Datagrams in the unit, or 0 if unit is null + */ + private int getNumberOfUnitFrames(SelectedUnit unit) { + int numberOfFrames = 0; + try { + Datagram[] frames = getUnitData(unit).getFrames(); + numberOfFrames = frames.length; + } catch (NullPointerException e) { + // leave at 0 + } + return numberOfFrames; + } + + /** + * Get the number of Datagrams in this phone's left unit + * + * @return the number of Datagrams in the left unit, or 0 if there is no left unit + */ + public int getNumberOfLeftUnitFrames() { + return getNumberOfUnitFrames(leftUnit); + } + + /** + * Get the number of Datagrams in this phone's right unit + * + * @return the number of Datagrams in the right unit, or 0 if there is no right unit + */ + public int getNumberOfRightUnitFrames() { + return getNumberOfUnitFrames(rightUnit); + } + + /** + * Get the number of Datagrams in this phone's left and right units + * + * @return the number of Datagrams in this phone + */ + public int getNumberOfFrames() { + return getNumberOfLeftUnitFrames() + getNumberOfRightUnitFrames(); + } + + /** + * Get the durations (in seconds) of each Datagram in this phone's units + * + * @return the left and right unit's Datagram durations (in seconds) in an array + */ + public double[] getFrameDurations() { + Datagram[] frames = getUnitDataFrames(); + double[] durations = new double[frames.length]; + for (int f = 0; f < frames.length; f++) { + durations[f] = frames[f].getDuration() / sampleRate; + } + return durations; + } + + /** + * Get the target duration (in seconds) of a HalfPhoneTarget + * + * @param target + * the target whose duration to get + * @return the target duration in seconds, or 0 if target is null + */ + private double getTargetDuration(HalfPhoneTarget target) { + double duration = 0; + try { + duration = target.getTargetDurationInSeconds(); + } catch (NullPointerException e) { + // leave at 0 + } + return duration; + } + + /** + * Get this phone's left target's duration (in seconds) + * + * @return the left target's duration in seconds, or 0 if there is no left target + */ + public double getLeftTargetDuration() { + return getTargetDuration(leftTarget); + } + + /** + * Get this phone's right target's duration (in seconds) + * + * @return the right target's duration in seconds, or 0 if there is no right target + */ + public double getRightTargetDuration() { + return getTargetDuration(rightTarget); + } + + /** + * Get the predicted duration of this phone (which is the sum of the left and right target's duration) + * + * @return the predicted phone duration in seconds + */ + public double getPredictedDuration() { + return getLeftTargetDuration() + getRightTargetDuration(); + } + + /** + * Convenience getter for a SelectedUnit's UnitData + * + * @param unit + * the unit whose UnitData to get + * @return the UnitData of the unit, or null, if unit is null + */ + private UnitData getUnitData(SelectedUnit unit) { + UnitData unitData = null; + try { + unitData = (UnitData) unit.getConcatenationData(); + } catch (NullPointerException e) { + // leave at null + } + return unitData; + } + + /** + * Get this phone's left unit's UnitData + * + * @return the left unit's UnitData, or null if there is no left unit + */ + public UnitData getLeftUnitData() { + return getUnitData(leftUnit); + } + + /** + * Get this phone's right unit's UnitData + * + * @return the right unit's UnitData, or null if there is no right unit + */ + public UnitData getRightUnitData() { + return getUnitData(rightUnit); + } + + /** + * Get the actual duration (in seconds) of a SelectedUnit, from its UnitData + * + * @param unit + * whose duration to get + * @return the unit's duration in seconds, or 0 if unit is null + */ + private double getUnitDuration(SelectedUnit unit) { + int durationInSamples = 0; + try { + durationInSamples = getUnitData(unit).getUnitDuration(); + } catch (NullPointerException e) { + // leave at 0 + } + double duration = durationInSamples / sampleRate; + return duration; + } + + /** + * Get this phone's left unit's duration (in seconds) + * + * @return the left unit's duration in seconds, or 0 if there is no left unit + */ + public double getLeftUnitDuration() { + return getUnitDuration(leftUnit); + } + + /** + * Get this phone's right unit's duration (in seconds) + * + * @return the right unit's duration in seconds, or 0 if there is no right unit + */ + public double getRightUnitDuration() { + return getUnitDuration(rightUnit); + } + + /** + * Get the realized duration (in seconds) of this phone (which is the sum of the durations of the left and right units) + * + * @return the phone's realized duration in seconds + */ + public double getRealizedDuration() { + return getLeftUnitDuration() + getRightUnitDuration(); + } + + /** + * Get the factor needed to convert the realized duration of a unit to the target duration + * + * @param unit + * whose realized duration to convert + * @param target + * whose duration to match + * @return the multiplication factor to convert from the realized duration to the target duration, or 0 if the unit duration + * is 0 + */ + private double getDurationFactor(SelectedUnit unit, HalfPhoneTarget target) { + double unitDuration = getUnitDuration(unit); + if (unitDuration <= 0) { + // throw new ArithmeticException("Realized duration must be greater than 0!"); + return 0; + } + double targetDuration = getTargetDuration(target); + double durationFactor = targetDuration / unitDuration; + return durationFactor; + } + + /** + * Get the factor to convert this phone's left unit's duration into this phone's left target duration + * + * @return the left duration factor + */ + public double getLeftDurationFactor() { + return getDurationFactor(leftUnit, leftTarget); + } + + /** + * Get the factor to convert this phone's right unit's duration into this phone's right target duration + * + * @return the right duration factor + */ + public double getRightDurationFactor() { + return getDurationFactor(rightUnit, rightTarget); + } + + /** + * Get the duration factors for this phone, one per datagram. Each factor corresponding to a datagram in the left unit is that + * required to convert the left unit's duration to the left target duration, and likewise for the right unit's datagrams. + * + * @return the left and right duration factors for this phone, in an array whose size matched the Datagrams in this phone's + * units + */ + public double[] getFramewiseDurationFactors() { + double[] durationFactors = new double[getNumberOfFrames()]; + int numberOfLeftUnitFrames = getNumberOfLeftUnitFrames(); + double leftDurationFactor = getLeftDurationFactor(); + Arrays.fill(durationFactors, 0, numberOfLeftUnitFrames, leftDurationFactor); + double rightDurationFactor = getRightDurationFactor(); + Arrays.fill(durationFactors, numberOfLeftUnitFrames, getNumberOfFrames(), rightDurationFactor); + return durationFactors; + } + + /** + * Set the target F0 values of this phone's left half, with one value per Datagram in the phone's left unit + * + * @param f0TargetValues + * array of target F0 values to assign to the left halfphone + * @throws IllegalArgumentException + * if the length of f0TargetValues does not match the number of Datagrams in the phone's left unit + */ + public void setLeftTargetF0Values(double[] f0TargetValues) throws IllegalArgumentException { + int numberOfLeftUnitFrames = getNumberOfLeftUnitFrames(); + if (f0TargetValues.length != numberOfLeftUnitFrames) { + throw new IllegalArgumentException("Wrong number of F0 targets (" + f0TargetValues.length + + ") for number of frames (" + numberOfLeftUnitFrames + " in halfphone: '" + leftUnit.toString() + "'"); + } + this.leftF0Targets = f0TargetValues; + } + + /** + * Set the target F0 values of this phone's right half, with one value per Datagram in the phone's right unit + * + * @param f0TargetValues + * array of target F0 values to assign to the right halfphone + * @throws IllegalArgumentException + * if the length of f0TargetValues does not match the number of Datagrams in the phone's right unit + */ + public void setRightTargetF0Values(double[] f0TargetValues) { + if (f0TargetValues.length != getNumberOfRightUnitFrames()) { + throw new IllegalArgumentException("Wrong number of F0 targets (" + f0TargetValues.length + + ") for number of frames (" + getNumberOfRightUnitFrames() + " in halfphone: '" + rightUnit.toString() + "'"); + } + this.rightF0Targets = f0TargetValues; + } + + /** + * Get the target F0 values for this phone's left half, with one value per Datagram in the phone's left unit + * + * @return the target F0 values for the left halfphone in an array + * @throws NullPointerException + * if the target F0 values for the left halfphone are null + */ + public double[] getLeftTargetF0Values() throws NullPointerException { + if (leftF0Targets == null) { + throw new NullPointerException("The left target F0 values have not been assigned!"); + } + return leftF0Targets; + } + + /** + * Get the target F0 values for this phone's right half, with one value per Datagram in the phone's right unit + * + * @return the target F0 values for the right halfphone in an array + * @throws NullPointerException + * if the target F0 values for the right halfphone are null + */ + public double[] getRightTargetF0Values() throws NullPointerException { + if (rightF0Targets == null) { + throw new NullPointerException("The right target F0 values have not been assigned!"); + } + return rightF0Targets; + } + + /** + * Get the target F0 values for this phone, with one value per Datagram in the phone's left and right units + * + * @return the target F0 values with one value per Datagram in an array + */ + public double[] getTargetF0Values() { + double[] f0Targets = ArrayUtils.addAll(leftF0Targets, rightF0Targets); + return f0Targets; + } + + /** + * Get the mean target F0 for this phone + * + * @return the mean predicted F0 value + */ + public double getPredictedF0() { + double meanF0 = MathUtils.mean(getTargetF0Values()); + return meanF0; + } + + /** + * Get the durations (in seconds) of each Datagram in a SelectedUnit's UnitData + * + * @param unit + * whose Datagrams' durations to get + * @return the durations (in seconds) of each Datagram in the unit in an array, or null if unit is null + */ + private double[] getUnitFrameDurations(SelectedUnit unit) { + Datagram[] frames = null; + try { + frames = getUnitData(unit).getFrames(); + } catch (NullPointerException e) { + return null; + } + assert frames != null; + + double[] frameDurations = new double[frames.length]; + for (int f = 0; f < frames.length; f++) { + long frameDuration = frames[f].getDuration(); + frameDurations[f] = frameDuration / sampleRate; // converting to seconds + } + return frameDurations; + } + + /** + * Get the durations (in seconds) of each Datagram in this phone's left unit + * + * @return the durations of each Datagram in the left unit in an array, or null if there is no left unit + */ + public double[] getLeftUnitFrameDurations() { + return getUnitFrameDurations(leftUnit); + } + + /** + * Get the durations (in seconds) of each Datagram in this phone's right unit + * + * @return the durations of each Datagram in the right unit in an array, or null if there is no right unit + */ + public double[] getRightUnitFrameDurations() { + return getUnitFrameDurations(rightUnit); + } + + /** + * Get the durations (in seconds) of each Datagram in this phone's left and right units + * + * @return the durations of all Datagrams in this phone in an array + */ + public double[] getRealizedFrameDurations() { + return ArrayUtils.addAll(getLeftUnitFrameDurations(), getRightUnitFrameDurations()); + } + + /** + * Get the F0 values from a SelectedUnit's Datagrams. + *

+ * Since these are not stored explicitly, we are forced to rely on the inverse of the Datagram durations, which means that + * during unvoiced regions, we recover a value of 100 Hz... + * + * @param unit + * from whose Datagrams to recover the F0 values + * @return the F0 value of each Datagram in the unit in an array, or null if the unit is null or any of the Datagrams have + * zero duration + */ + private double[] getUnitF0Values(SelectedUnit unit) { + double[] f0Values = null; + try { + double[] durations = getUnitFrameDurations(unit); + if (!ArrayUtils.contains(durations, 0)) { + f0Values = MathUtils.invert(durations); + } else { + // leave at null + } + } catch (IllegalArgumentException e) { + // leave at null + } + return f0Values; + } + + /** + * Recover the F0 values from this phone's left unit's Datagrams + * + * @return the left unit's F0 values in an array, with one value per Datagram, or null if there is no left unit or any of the + * left unit's Datagrams have zero duration + */ + public double[] getLeftUnitFrameF0s() { + return getUnitF0Values(leftUnit); + } + + /** + * Recover the F0 values from this phone's right unit's Datagrams + * + * @return the right unit's F0 values in an array, with one value per Datagram, or null if there is no right unit or any of + * the right unit's Datagrams have zero duration + */ + public double[] getRightUnitFrameF0s() { + return getUnitF0Values(rightUnit); + } + + /** + * Recover the F0 values from each Datagram in this phone's left and right units + * + * @return the F0 values for each Datagram in this phone, or null if either the left or the right unit contain a Datagram with + * zero duration + */ + public double[] getUnitFrameF0s() { + double[] leftUnitFrameF0s = getLeftUnitFrameF0s(); + double[] rightUnitFrameF0s = getRightUnitFrameF0s(); + double[] unitFrameF0s = ArrayUtils.addAll(leftUnitFrameF0s, rightUnitFrameF0s); + return unitFrameF0s; + } + + /** + * Get the realized F0 by recovering the F0 from all Datagrams in this phone and computing the mean + * + * @return the mean F0 of all Datagrams in this phone's left and right units + */ + public double getRealizedF0() { + double meanF0 = MathUtils.mean(getUnitFrameF0s()); + return meanF0; + } + + /** + * Get the factors required to convert the F0 values recovered from the Datagrams in a SelectedUnit to the target F0 values. + * + * @param unit + * from which to recover the realized F0 values + * @param target + * for which to get the target F0 values + * @return each Datagram's F0 factor in an array, or null if realized and target F0 values differ in length + * @throws ArithmeticException + * if any of the F0 values recovered from the unit's Datagrams is zero + */ + private double[] getUnitF0Factors(SelectedUnit unit, HalfPhoneTarget target) throws ArithmeticException { + double[] unitF0Values = getUnitF0Values(unit); + if (ArrayUtils.contains(unitF0Values, 0)) { + throw new ArithmeticException("Unit frames must not have F0 of 0!"); + } + + double[] targetF0Values; + if (target == null || target.isLeftHalf()) { + targetF0Values = getLeftTargetF0Values(); + } else { + targetF0Values = getRightTargetF0Values(); + } + + double[] f0Factors = null; + try { + f0Factors = MathUtils.divide(targetF0Values, unitF0Values); + } catch (IllegalArgumentException e) { + // leave at null + } + return f0Factors; + } + + /** + * Get the factors to convert each of the F0 values in this phone's left half to the corresponding target value + * + * @return the F0 factors for this phone's left half in an array with one value per Datagram, or null if the number of + * Datagrams does not match the number of left target F0 values + */ + public double[] getLeftF0Factors() { + return getUnitF0Factors(leftUnit, leftTarget); + } + + /** + * Get the factors to convert each of the F0 values in this phone's right half to the corresponding target value + * + * @return the F0 factors for this phone's right half in an array with one value per Datagram, or null if the number of + * Datagrams does not match the number of right target F0 values + */ + public double[] getRightF0Factors() { + return getUnitF0Factors(rightUnit, rightTarget); + } + + /** + * Get the F0 factor for each Datagram in this phone's left and right units + * + * @return the F0 factors, in an array with one value per Datagram + */ + public double[] getF0Factors() { + return ArrayUtils.addAll(getLeftF0Factors(), getRightF0Factors()); + } + + /** + * Get the Allophone represented by this + * + * @return the Allophone + */ + private Allophone getAllophone() { + if (leftTarget != null) { + return leftTarget.getAllophone(); + } else if (rightTarget != null) { + return rightTarget.getAllophone(); + } + return null; + } + + /** + * Determine whether this is a transient phone (i.e. a plosive) + * + * @return true if this is a plosive, false otherwise + */ + public boolean isTransient() { + Allophone allophone = getAllophone(); + if (allophone.isPlosive() || allophone.isAffricate()) { + return true; + } else { + return false; + } + } + + /** + * Determine whether this is a voiced phone + * + * @return true if this is voiced, false otherwise + */ + public boolean isVoiced() { + Allophone allophone = getAllophone(); + if (allophone.isVoiced()) { + return true; + } else { + return false; + } + } + + /** + * get the MaryXML Element corresponding to this Phone's Target + *

+ * If both leftTarget and rightTarget are not null, their respective MaryXML Elements should be + * equal. + * + * @return the MaryXML Element, or null if both halfphone targets are null + */ + public Element getMaryXMLElement() { + if (leftTarget != null) { + return leftTarget.getMaryxmlElement(); + } else if (rightTarget != null) { + return rightTarget.getMaryxmlElement(); + } + return null; + } + + /** + * for debugging, provide the names of the left and right targets as the string representation of this class + */ + public String toString() { + String string = ""; + if (leftTarget != null) { + string += " " + leftTarget.getName(); + } + if (rightTarget != null) { + string += " " + rightTarget.getName(); + } + return string; + } +} diff --git a/marytts-unitselection/src/main/java/marytts/unitselection/analysis/ProsodyAnalyzer.java b/marytts-unitselection/src/main/java/marytts/unitselection/analysis/ProsodyAnalyzer.java new file mode 100644 index 0000000000..6f073932a5 --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/unitselection/analysis/ProsodyAnalyzer.java @@ -0,0 +1,437 @@ +/** + * Copyright 2010 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ + +package marytts.unitselection.analysis; + +import java.io.File; +import java.io.IOException; +import java.io.PrintWriter; +import java.util.ArrayList; +import java.util.List; +import java.util.ListIterator; + +import marytts.datatypes.MaryXML; +import marytts.modules.acoustic.ProsodyElementHandler; +import marytts.features.HalfPhoneTarget; +import marytts.unitselection.select.SelectedUnit; +import marytts.util.MaryUtils; + +import org.apache.commons.lang.ArrayUtils; +import org.apache.log4j.Logger; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.NodeList; + +/** + * Class to provide high-level, phone-based access to the predicted and realized prosodic parameters in a given unit-selection + * result + * + * @author steiner + * + */ +public class ProsodyAnalyzer { + + private List units; + + private int sampleRate; + + private Logger logger; + + private List phones; + + /** + * Main constructor + *

+ * Note that the units are first parsed into phones (and the F0 target values assigned), before any distinction is made + * between those with and without a realized duration (e.g. {@link #getRealizedPhones()}). + * + * @param units + * whose predicted and realized prosody to analyze + * @param sampleRate + * of the unit database, in Hz + * @throws Exception + * if the units cannot be parsed into phones + */ + public ProsodyAnalyzer(List units, int sampleRate) throws Exception { + this.units = units; + this.sampleRate = sampleRate; + + this.logger = MaryUtils.getLogger(this.getClass()); + + // List of phone segments: + this.phones = parseIntoPhones(); + } + + /** + * Parse a list of selected units into the corresponding phone segments + * + * @return List of Phones + * @throws Exception + * if the predicted prosody cannot be determined properly + */ + private List parseIntoPhones() throws Exception { + // initialize List of Phones (note that initial size is not final!): + phones = new ArrayList(units.size() / 2); + // iterate over the units: + int u = 0; + while (u < units.size()) { + // get unit... + SelectedUnit unit = units.get(u); + // ...and its target as a HalfPhoneTarget, so that we can... + HalfPhoneTarget target = (HalfPhoneTarget) unit.getTarget(); + // ...query its position in the phone: + if (target.isLeftHalf()) { + // if this is the left half of a phone... + if (u < units.size() - 1) { + // ...and there is a next unit in the list... + SelectedUnit nextUnit = units.get(u + 1); + HalfPhoneTarget nextTarget = (HalfPhoneTarget) nextUnit.getTarget(); + if (nextTarget.isRightHalf()) { + // ...and the next unit's target is the right half of the phone, add the phone: + phones.add(new Phone(unit, nextUnit, sampleRate)); + u++; + } else { + // otherwise, add a degenerate phone with no right halfphone: + phones.add(new Phone(unit, null, sampleRate)); + } + } else { + // otherwise, add a degenerate phone with no right halfphone: + phones.add(new Phone(unit, null, sampleRate)); + } + } else { + // otherwise, add a degenerate phone with no left halfphone: + phones.add(new Phone(null, unit, sampleRate)); + } + u++; + } + + // make sure we've seen all the units: + assert u == units.size(); + + // assign target F0 values to Phones: + insertTargetF0Values(); + + return phones; + } + + /** + * Assign predicted F0 values to the phones by parsing the XML Document + * + * @throws Exception + * if the Document cannot be accessed + */ + private void insertTargetF0Values() throws Exception { + NodeList phoneNodes; + try { + phoneNodes = getPhoneNodes(); + } catch (Exception e) { + throw new Exception("Could not get the phone Nodes from the Document", e); + } + + // count the number of Datagrams we need, which is the number of F0 target values the ProsodyElementHandler will return: + int totalNumberOfFrames = getNumberOfFrames(); + + // this method hinges on the F0 attribute parsing done in modules.acoustic + ProsodyElementHandler elementHandler = new ProsodyElementHandler(); + double[] f0Targets = elementHandler.getF0Contour(phoneNodes, totalNumberOfFrames); + + int f0TargetStartIndex = 0; + for (Phone phone : phones) { + int numberOfLeftUnitFrames = phone.getNumberOfLeftUnitFrames(); + int f0TargetMidIndex = f0TargetStartIndex + numberOfLeftUnitFrames; + double[] leftF0Targets = ArrayUtils.subarray(f0Targets, f0TargetStartIndex, f0TargetMidIndex); + phone.setLeftTargetF0Values(leftF0Targets); + + int numberOfRightUnitFrames = phone.getNumberOfRightUnitFrames(); + int f0TargetEndIndex = f0TargetMidIndex + numberOfRightUnitFrames; + double[] rightF0Targets = ArrayUtils.subarray(f0Targets, f0TargetMidIndex, f0TargetEndIndex); + phone.setRightTargetF0Values(rightF0Targets); + + f0TargetStartIndex = f0TargetEndIndex; + } + return; + } + + /** + * Get the List of Phones + * + * @return the Phones + */ + public List getPhones() { + return phones; + } + + /** + * Get the List of Phones that have a predicted duration greater than zero + * + * @return the List of realized Phones + */ + public List getRealizedPhones() { + List realizedPhones = new ArrayList(phones.size()); + for (Phone phone : phones) { + if (phone.getPredictedDuration() > 0) { + realizedPhones.add(phone); + } + } + return realizedPhones; + } + + /** + * Get NodeList for Phones from Document + * + * @return NodeList of Phones + * @throws Exception + * if Document cannot be accessed + */ + private NodeList getPhoneNodes() throws Exception { + Document document = getDocument(); + NodeList phoneNodes; + try { + phoneNodes = document.getElementsByTagName(MaryXML.PHONE); + } catch (NullPointerException e) { + throw new Exception("Could not access the Document!", e); + } + return phoneNodes; + } + + /** + * For the first phone with a MaryXMLElement we encounter, return that Element's Document + * + * @return the Document containing the {@link #phones} or null if no phone is able to provide a MaryXMLElement + */ + private Document getDocument() { + for (Phone phone : phones) { + Element phoneElement = phone.getMaryXMLElement(); + if (phoneElement != null) { + return phoneElement.getOwnerDocument(); + } + } + return null; + } + + /** + * Get the number of Datagrams in all Phones + * + * @return the number of Datagrams in all Phones + */ + private int getNumberOfFrames() { + int totalNumberOfFrames = 0; + for (Phone phone : phones) { + totalNumberOfFrames += phone.getNumberOfFrames(); + } + return totalNumberOfFrames; + } + + /** + * Get duration factors representing ratio of predicted and realized halfphone Unit durations. Units with zero predicted or + * realized duration receive a factor of 0. + * + * @return List of duration factors + */ + public List getDurationFactors() { + // list of duration factors, one per halfphone unit: + List durationFactors = new ArrayList(units.size()); + + // iterate over phone segments: + for (Phone phone : phones) { + double leftDurationFactor = phone.getLeftDurationFactor(); + if (leftDurationFactor > 0) { + durationFactors.add(leftDurationFactor); + logger.debug("duration factor for unit " + phone.getLeftUnit().getTarget().getName() + " -> " + + leftDurationFactor); + } + double rightDurationFactor = phone.getRightDurationFactor(); + if (rightDurationFactor > 0) { + // ...add the duration factor to the list: + durationFactors.add(rightDurationFactor); + logger.debug("duration factor for unit " + phone.getRightUnit().getTarget().getName() + " -> " + + rightDurationFactor); + } + } + + return durationFactors; + } + + /* + * Some ad-hoc methods for HnmUnitConcatenator: + */ + + public double[] getDurationFactorsFramewise() { + double[] f0Factors = null; + for (Phone phone : phones) { + double[] phoneF0Factors = phone.getFramewiseDurationFactors(); + f0Factors = ArrayUtils.addAll(f0Factors, phoneF0Factors); + } + return f0Factors; + } + + public double[] getFrameMidTimes() { + double[] frameDurations = null; + for (Phone phone : phones) { + double[] phoneFrameDurations = phone.getFrameDurations(); + frameDurations = ArrayUtils.addAll(frameDurations, phoneFrameDurations); + } + + assert frameDurations != null; + double[] frameMidTimes = new double[frameDurations.length]; + double frameStartTime = 0; + for (int f = 0; f < frameDurations.length; f++) { + frameMidTimes[f] = frameStartTime + frameDurations[f] / 2; + frameStartTime += frameDurations[f]; + } + + return frameMidTimes; + } + + public double[] getF0Factors() { + double[] f0Factors = null; + for (Phone phone : phones) { + double[] phoneF0Factors = phone.getF0Factors(); + f0Factors = ArrayUtils.addAll(f0Factors, phoneF0Factors); + } + return f0Factors; + } + + /** + * For debugging, generate Praat DurationTier, which can be used for PSOLA-based manipulation in Praat. + *

+ * Notes: + *

    + *
  • Initial silence is skipped.
    • + *
  • Units with zero realized duration are ignored.
    • + *
  • To avoid gradual interpolation between points, place two points around each unit boundary, separated by + * MIN_SKIP; this workaround allows one constant factor per unit.
    • + *
+ * + * @param fileName + * of the DurationTier to be generated + * @throws IOException + */ + public void writePraatDurationTier(String fileName) throws IOException { + + // initialize times and values with a size corresponding to two elements (start and end) per unit: + ArrayList times = new ArrayList(units.size() * 2); + ArrayList values = new ArrayList(units.size() * 2); + + final double MIN_SKIP = 1e-15; + + // cumulative time pointer: + double time = 0; + + // iterate over phones, skipping the initial silence: + // TODO is this really robust? + ListIterator phoneIterator = phones.listIterator(1); + while (phoneIterator.hasNext()) { + Phone phone = phoneIterator.next(); + + // process left halfphone unit: + if (phone.getLeftUnitDuration() > 0) { + // add point at unit start: + times.add(time); + values.add(phone.getLeftDurationFactor()); + + // increment time pointer by unit duration: + time += phone.getLeftUnitDuration(); + + // add point at unit end: + times.add(time - MIN_SKIP); + values.add(phone.getLeftDurationFactor()); + } + // process right halfphone unit: + if (phone.getRightUnitDuration() > 0) { + // add point at unit start: + times.add(time); + values.add(phone.getRightDurationFactor()); + + // increment time pointer by unit duration: + time += phone.getRightUnitDuration(); + + // add point at unit end: + times.add(time - MIN_SKIP); + values.add(phone.getRightDurationFactor()); + } + } + + // open file for writing: + File durationTierFile = new File(fileName); + PrintWriter out = new PrintWriter(durationTierFile); + + // print header: + out.println("\"ooTextFile\""); + out.println("\"DurationTier\""); + out.println(String.format("0 %f %d", time, times.size())); + + // print points (times and values): + for (int i = 0; i < times.size(); i++) { + // Note: time precision should be greater than MIN_SKIP: + out.println(String.format("%.16f %f", times.get(i), values.get(i))); + } + + // flush and close: + out.close(); + } + + /** + * For debugging, generate Praat PitchTier, which can be used for PSOLA-based manipulation in Praat. + * + * @param fileName + * of the PitchTier to be generated + * @throws IOException + */ + public void writePraatPitchTier(String fileName) throws IOException { + + // initialize times and values: + ArrayList times = new ArrayList(); + ArrayList values = new ArrayList(); + + // cumulative time pointer: + double time = 0; + + // iterate over phones, skipping the initial silence: + ListIterator phoneIterator = phones.listIterator(1); + while (phoneIterator.hasNext()) { + Phone phone = phoneIterator.next(); + double[] frameTimes = phone.getRealizedFrameDurations(); + double[] frameF0s = phone.getUnitFrameF0s(); + for (int f = 0; f < frameF0s.length; f++) { + time += frameTimes[f]; + times.add(time); + values.add(frameF0s[f]); + } + } + + // open file for writing: + File durationTierFile = new File(fileName); + PrintWriter out = new PrintWriter(durationTierFile); + + // print header: + out.println("\"ooTextFile\""); + out.println("\"PitchTier\""); + out.println(String.format("0 %f %d", time, times.size())); + + // print points (times and values): + for (int i = 0; i < times.size(); i++) { + out.println(String.format("%.16f %f", times.get(i), values.get(i))); + } + + // flush and close: + out.close(); + } +} diff --git a/marytts-unitselection/src/main/java/marytts/unitselection/analysis/VoiceDataDumper.java b/marytts-unitselection/src/main/java/marytts/unitselection/analysis/VoiceDataDumper.java new file mode 100644 index 0000000000..f1b1b033a5 --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/unitselection/analysis/VoiceDataDumper.java @@ -0,0 +1,351 @@ +/** + * Copyright 2010 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ + +package marytts.unitselection.analysis; + +import java.io.BufferedOutputStream; +import java.io.BufferedWriter; +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.PrintWriter; +import java.nio.BufferUnderflowException; + +import marytts.exceptions.MaryConfigurationException; +import marytts.features.FeatureDefinition; +import marytts.features.FeatureVector; +import marytts.server.MaryProperties; +import marytts.unitselection.data.FeatureFileReader; +import marytts.unitselection.data.TimelineReader; +import marytts.unitselection.data.Unit; +import marytts.unitselection.data.UnitDatabase; +import marytts.unitselection.data.UnitFileReader; +import marytts.util.data.Datagram; +import marytts.util.data.text.PraatInterval; +import marytts.util.data.text.PraatIntervalTier; +import marytts.util.data.text.PraatTextGrid; + +/** + * Convenience class to dump relevant data from a unit selection voice to a Praat TextGrid and a wav file for inspection of + * timeline data in external tools (e.g. Praat, WaveSurfer, etc.) + * + * @author steiner + * + */ +public class VoiceDataDumper { + protected UnitDatabase unitDB; + + protected FeatureFileReader featureFileReader; + + protected long numSamples = 0; + + protected FeatureDefinition featureDefinition; + + protected int phoneFeatureIndex; + + protected int halfphoneLRFeatureIndex; + + public VoiceDataDumper() { + + } + + /** + * @see marytts.util.data.audio.WavWriter#byteswap(int) + */ + protected int byteswap(int val) { + return (((val & 0xff000000) >>> 24) + ((val & 0x00ff0000) >>> 8) + ((val & 0x0000ff00) << 8) + ((val & 0x000000ff) << 24)); + } + + /** + * @see marytts.util.data.audio.WavWriter#byteswap(short) + */ + protected short byteswap(short val) { + return ((short) ((((int) (val) & 0xff00) >>> 8) + (((int) (val) & 0x00ff) << 8))); + } + + /** + * Load audio timeline from file + * + * @param fileName + * to load + * @return TimelineReader + * @throws IOException + */ + protected TimelineReader loadAudioTimeline(String fileName) throws IOException, MaryConfigurationException { + return new TimelineReader(fileName); + } + + /** + * Load unit database from various relevant files + * + * @param audioTimelineFileName + * to load + * @param basenameTimelineFileName + * to load + * @param unitFileName + * to load + * @throws IOException + */ + protected void loadUnitDatabase(String audioTimelineFileName, String basenameTimelineFileName, String unitFileName) + throws IOException, MaryConfigurationException { + unitDB = new UnitDatabase(); + UnitFileReader unitFileReader = new UnitFileReader(unitFileName); + TimelineReader audioTimelineReader = loadAudioTimeline(audioTimelineFileName); + TimelineReader basenameTimelineReader = new TimelineReader(basenameTimelineFileName); + unitDB.load(null, null, unitFileReader, null, audioTimelineReader, basenameTimelineReader, 0); + } + + /** + * Load unit feature file from file + * + * @param fileName + * to load + * @throws IOException + */ + protected void loadFeatureFile(String fileName) throws IOException, MaryConfigurationException { + featureFileReader = new FeatureFileReader(fileName); + featureDefinition = featureFileReader.getFeatureDefinition(); + phoneFeatureIndex = featureDefinition.getFeatureIndex("phone"); + halfphoneLRFeatureIndex = featureDefinition.getFeatureIndex("halfphone_lr"); + } + + /** + * Get total duration of a Datagram array + * + * @param datagrams + * whose duration to get + * @return total duration in seconds + */ + protected double getDuration(Datagram[] datagrams) { + double totalDuration = 0; + for (Datagram datagram : datagrams) { + totalDuration += datagram.getDuration() / (float) unitDB.getAudioTimeline().getSampleRate(); + } + return totalDuration; + } + + /** + * Get raw samples from all Datagrams in an array + * + * @param datagrams + * whose samples to get + * @return raw samples as stored in the Datagrams + * @throws IOException + */ + protected byte[] getSamples(Datagram[] datagrams) throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + for (Datagram datagram : datagrams) { + byte[] data = datagram.getData(); + baos.write(data); + } + byte[] samples = baos.toByteArray(); + return samples; + } + + /** + * Dump units to Praat TextGrid. This will have three tiers: + *
    + *
  1. halfphone units, labeled with unit indices;
    2. + *
  2. phone units, labeled with allophones;
    3. + *
  3. basenames, labeled with basename of original utterance.
    4. + *
+ * + * @param fileName + * of new TextGrid + * @throws IOException + * if data files cannot be read, or TextGrid cannot be written + */ + protected void dumpTextGrid(String fileName) throws IOException { + // init the tiers: + PraatIntervalTier unitTier = new PraatIntervalTier("unitindex"); + PraatIntervalTier phoneTier = new PraatIntervalTier("halfphone"); + PraatIntervalTier basenameTier = new PraatIntervalTier("basename"); + + // init some variables: + double prevHalfPhoneUnitDurationInSeconds = 0; + double basenameDurationInSeconds = 0; + String basenameLabel = null; + + // iterate over all units: + for (int unitIndex = 0; unitIndex < unitDB.getUnitFileReader().getNumberOfUnits(); unitIndex++) { + // if (unitIndex > 727) { + // break; + // } + Unit unit = unitDB.getUnitFileReader().getUnit(unitIndex); + if (unit.isEdgeUnit()) { + // if this is the left edge, basenameDurationInSeconds will be 0 + if (basenameDurationInSeconds > 0) { + // add basename interval + PraatInterval basenameInterval = new PraatInterval(basenameDurationInSeconds, basenameLabel); + basenameTier.appendInterval(basenameInterval); + basenameDurationInSeconds = 0; + } + continue; // ignore edge units (also, avoid ticket:335) + } + + // iterate over datagrams to get exact duration: + Datagram[] datagrams; + try { + datagrams = unitDB.getAudioTimeline().getDatagrams(unit, unitDB.getAudioTimeline().getSampleRate()); + } catch (BufferUnderflowException e) { + throw e; + } + double halfPhoneUnitDurationInSeconds = getDuration(datagrams); + // cumulative sample count for wav file header: + byte[] buf = getSamples(datagrams); + numSamples += buf.length; + + // keep track of basename duration and label: + basenameDurationInSeconds += halfPhoneUnitDurationInSeconds; + basenameLabel = unitDB.getFilename(unit); + + // halfphone unit interval (labeled with unit index): + PraatInterval interval = new PraatInterval(halfPhoneUnitDurationInSeconds, Integer.toString(unit.index)); + unitTier.appendInterval(interval); + + // lazy way of checking that we have both halves of the phone: + FeatureVector features = featureFileReader.getFeatureVector(unit); + String halfphoneLR = features.getFeatureAsString(halfphoneLRFeatureIndex, featureDefinition); + if (halfphoneLR.equals("R")) { + // phone interval: + double phoneUnitDurationInSeconds = halfPhoneUnitDurationInSeconds + prevHalfPhoneUnitDurationInSeconds; + String phoneLabel = features.getFeatureAsString(phoneFeatureIndex, featureDefinition); + PraatInterval phoneInterval = new PraatInterval(phoneUnitDurationInSeconds, phoneLabel); + phoneTier.appendInterval(phoneInterval); + } + prevHalfPhoneUnitDurationInSeconds = halfPhoneUnitDurationInSeconds; + } + + // update time domains: + unitTier.updateBoundaries(); + phoneTier.updateBoundaries(); + basenameTier.updateBoundaries(); + + // create TextGrid: + PraatTextGrid textGrid = new PraatTextGrid(); + textGrid.appendTier(unitTier); + textGrid.appendTier(phoneTier); + textGrid.appendTier(basenameTier); + + // write to text file: + BufferedWriter output = new BufferedWriter(new PrintWriter(fileName)); + output.write(textGrid.toString()); + output.close(); + } + + /** + * Adapted from {@link marytts.util.data.audio.WavWriter#export(String, int, byte[])} and + * {@link marytts.util.data.audio.WavWriter#doWrite(String, int)} + */ + protected void dumpAudio(String fileName) throws IOException { + // refuse to write wav file if we don't know how many samples there are: + if (!(numSamples > 0)) { + return; + } + + // open wav file, and write header: + DataOutputStream dos = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(fileName))); + int nBytesPerSample = 2; + dos.writeBytes("RIFF"); // "RIFF" in ascii + dos.writeInt(byteswap((int) (36 + numSamples))); // Chunk size + dos.writeBytes("WAVEfmt "); + dos.writeInt(byteswap(16)); // chunk size, 16 for PCM + dos.writeShort(byteswap((short) 1)); // PCM format + dos.writeShort(byteswap((short) 1)); // Mono, one channel + dos.writeInt(byteswap(unitDB.getAudioTimeline().getSampleRate())); // Samplerate + dos.writeInt(byteswap(unitDB.getAudioTimeline().getSampleRate() * nBytesPerSample)); // Byte-rate + dos.writeShort(byteswap((short) (nBytesPerSample))); // Nbr of bytes per samples x nbr of channels + dos.writeShort(byteswap((short) (nBytesPerSample * 8))); // nbr of bits per sample + dos.writeBytes("data"); + dos.writeInt(byteswap((int) numSamples)); + + // implicitly unit-wise buffered writing of samples: + for (int unitIndex = 0; unitIndex < unitDB.getUnitFileReader().getNumberOfUnits(); unitIndex++) { + Unit unit = unitDB.getUnitFileReader().getUnit(unitIndex); + if (unit.isEdgeUnit()) { + continue; // ignore edge units (also, avoid ticket:335) + } + + Datagram[] datagrams = unitDB.getAudioTimeline().getDatagrams(unit, unitDB.getAudioTimeline().getSampleRate()); + byte[] buf = getSamples(datagrams); + + // write buffer to file: + // Byte-swap the samples + byte b = 0; + for (int j = 0; j < buf.length - 1; j += 2) { + b = buf[j]; + try { + buf[j] = buf[j + 1]; + } catch (ArrayIndexOutOfBoundsException e) { + throw e; + } + buf[j + 1] = b; + } + dos.write(buf); + } + + dos.close(); + } + + /** + * Get file names from voice config file. Dump relevant data from audio timeline, unit file, etc. to Praat TextGrid and wav + * file. + * + * @param voiceName + * for config file to read (e.g. "bits3") + * @throws Exception + */ + protected void dumpData(String voiceName) throws Exception { + + String audioTimelineFileName = MaryProperties.needFilename("voice." + voiceName + ".audioTimelineFile"); + String basenameTimelineFileName = MaryProperties.needFilename("voice." + voiceName + ".basenameTimeline"); + String unitFileName = MaryProperties.needFilename("voice." + voiceName + ".unitsFile"); + String featureFileName = MaryProperties.needFilename("voice." + voiceName + ".featureFile"); + String textGridFilename = audioTimelineFileName.replace(".mry", ".TextGrid"); + String wavFilename = audioTimelineFileName.replace(".mry", ".wav"); + + loadUnitDatabase(audioTimelineFileName, basenameTimelineFileName, unitFileName); + loadFeatureFile(featureFileName); + System.out.println("All files loaded."); + dumpTextGrid(textGridFilename); + System.out.println("Dumped TextGrid to " + textGridFilename); + dumpAudio(wavFilename); + System.out.println("Dumped audio to " + wavFilename); + } + + /** + * Main method. Add VOICE jar to classpath, then call with + * + * 
+	 * -ea -Xmx1gb -Dmary.base=$MARYBASE VOICE
+	 *
+ * + * or something similar + * + * @param args + * voice name (without the Locale) of voice to dump data from + * @throws Exception + */ + public static void main(String[] args) throws Exception { + new VoiceDataDumper().dumpData(args[0]); + } + +} diff --git a/marytts-unitselection/src/main/java/marytts/unitselection/concat/BaseUnitConcatenator.java b/marytts-unitselection/src/main/java/marytts/unitselection/concat/BaseUnitConcatenator.java new file mode 100644 index 0000000000..e708236035 --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/unitselection/concat/BaseUnitConcatenator.java @@ -0,0 +1,315 @@ +/** + * Portions Copyright 2006 DFKI GmbH. + * Portions Copyright 2001 Sun Microsystems, Inc. + * Portions Copyright 1999-2001 Language Technologies Institute, + * Carnegie Mellon University. + * All Rights Reserved. Use is subject to license terms. + * + * Permission is hereby granted, free of charge, to use and distribute + * this software and its documentation without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of this work, and to + * permit persons to whom this work is furnished to do so, subject to + * the following conditions: + * + * 1. The code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * 2. Any modifications must be clearly marked as such. + * 3. Original authors' names are not deleted. + * 4. The authors' names are not used to endorse or promote products + * derived from this software without specific prior written + * permission. + * + * DFKI GMBH AND THE CONTRIBUTORS TO THIS WORK DISCLAIM ALL WARRANTIES WITH + * REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL DFKI GMBH NOR THE + * CONTRIBUTORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL + * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR + * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS + * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF + * THIS SOFTWARE. + */ +package marytts.unitselection.concat; + +import java.io.IOException; +import java.util.Arrays; +import java.util.LinkedList; +import java.util.List; + +import javax.sound.sampled.AudioFormat; +import javax.sound.sampled.AudioInputStream; + +import marytts.unitselection.analysis.ProsodyAnalyzer; +import marytts.unitselection.data.TimelineReader; +import marytts.unitselection.data.UnitDatabase; +import marytts.unitselection.select.SelectedUnit; +import marytts.util.MaryUtils; +import marytts.util.data.BufferedDoubleDataSource; +import marytts.util.data.Datagram; +import marytts.util.data.DatagramDoubleDataSource; +import marytts.util.data.DoubleDataSource; +import marytts.util.data.audio.DDSAudioInputStream; + +import org.apache.log4j.Logger; + +/** + * Concatenates Units and returns an audio stream + * + * + */ +public class BaseUnitConcatenator implements UnitConcatenator { + protected Logger logger; + protected UnitDatabase database; + protected TimelineReader timeline; + protected AudioFormat audioformat; + protected double unitToTimelineSampleRateFactor; + + protected ProsodyAnalyzer prosodyAnalyzer; + + /** + * Empty Constructor; need to call load(UnitDatabase) separately + * + * @see #load(UnitDatabase) + */ + public BaseUnitConcatenator() { + logger = MaryUtils.getLogger(this.getClass()); + } + + public void load(UnitDatabase unitDatabase) { + this.database = unitDatabase; + this.timeline = database.getAudioTimeline(); + int sampleRate = timeline.getSampleRate(); + this.audioformat = new AudioFormat(AudioFormat.Encoding.PCM_SIGNED, sampleRate, // samples per second + 16, // bits per sample + 1, // mono + 2, // nr. of bytes per frame + sampleRate, // nr. of frames per second + true); // big-endian; + this.unitToTimelineSampleRateFactor = sampleRate / (double) database.getUnitFileReader().getSampleRate(); + } + + /** + * Provide the audio format which will be produced by this unit concatenator. + * + * @return the audio format + */ + public AudioFormat getAudioFormat() { + return audioformat; + } + + /** + * Build the audio stream from the units + * + * @param units + * the units + * @return the resulting audio stream + */ + public AudioInputStream getAudio(List units) throws IOException { + logger.debug("Getting audio for " + units.size() + " units"); + + // 1. Get the raw audio material for each unit from the timeline + getDatagramsFromTimeline(units); + + // 2. Determine target pitchmarks (= duration and f0) for each unit + determineTargetPitchmarks(units); + + // 2a. Analyze SelectedUnits wrt predicted vs. realized prosody + try { + prosodyAnalyzer = new ProsodyAnalyzer(units, timeline.getSampleRate()); + } catch (Exception e) { + throw new IOException("Could not analyze prosody!", e); + } + + // 3. Generate audio to match the target pitchmarks as closely as possible + return generateAudioStream(units); + } + + /** + * Get the raw audio material for each unit from the timeline. + * + * @param units + */ + protected void getDatagramsFromTimeline(List units) throws IOException { + for (SelectedUnit unit : units) { + UnitData unitData = new UnitData(); + unit.setConcatenationData(unitData); + int nSamples = 0; + int unitSize = unitToTimeline(unit.getUnit().duration); // convert to timeline samples + long unitStart = unitToTimeline(unit.getUnit().startTime); // convert to timeline samples + // System.out.println("Unit size "+unitSize+", pitchmarksInUnit "+pitchmarksInUnit); + Datagram[] datagrams = timeline.getDatagrams(unitStart, (long) unitSize); + unitData.setFrames(datagrams); + } + } + + /** + * Determine target pitchmarks (= duration and f0) for each unit. + * + * @param units + */ + protected void determineTargetPitchmarks(List units) { + for (SelectedUnit unit : units) { + UnitData unitData = (UnitData) unit.getConcatenationData(); + assert unitData != null : "Should not have null unitdata here"; + Datagram[] datagrams = unitData.getFrames(); + Datagram[] frames = null; // frames to realise + // The number and duration of the frames to realise + // must be the result of the target pitchmark computation. + if (datagrams != null && datagrams.length > 0) { + frames = datagrams; + } else { // no datagrams -- set as silence + int targetLength = (int) (unit.getTarget().getTargetDurationInSeconds() * timeline.getSampleRate()); + frames = new Datagram[] { createZeroDatagram(targetLength) }; + } + int unitDuration = 0; + for (int i = 0; i < frames.length; i++) { + int dur = (int) frames[i].getDuration(); + unitDuration += frames[i].getDuration(); + } + + unitData.setUnitDuration(unitDuration); + unitData.setFrames(frames); + } + } + + /** + * Generate audio to match the target pitchmarks as closely as possible. + * + * @param units + * @return + * @throws IOException + */ + protected AudioInputStream generateAudioStream(List units) throws IOException { + LinkedList datagrams = new LinkedList(); + for (SelectedUnit unit : units) { + UnitData unitData = (UnitData) unit.getConcatenationData(); + assert unitData != null : "Should not have null unitdata here"; + Datagram[] frames = unitData.getFrames(); + assert frames != null : "Cannot generate audio from null frames"; + // Generate audio from frames + datagrams.addAll(Arrays.asList(frames)); + } + + DoubleDataSource audioSource = new DatagramDoubleDataSource(datagrams); + return new DDSAudioInputStream(new BufferedDoubleDataSource(audioSource), audioformat); + } + + /** + * Create a datagram appropriate for this unit concatenator which contains only zero values as samples. + * + * @param length + * the number of zeros that the datagram should contain + * @return + */ + protected Datagram createZeroDatagram(int length) { + return new Datagram(length, new byte[2 * length]); + } + + protected int unitToTimeline(int duration) { + return (int) (duration * unitToTimelineSampleRateFactor); + } + + protected long unitToTimeline(long time) { + return (long) (time * unitToTimelineSampleRateFactor); + } + + public static class UnitData { + protected int[] pitchmarks; + protected Datagram[] frames; + protected Datagram rightContextFrame; + + protected int unitDuration = -1; + + public UnitData() { + } + + /** + * Set the array of to-be-realised pitchmarks for the realisation of the selected unit. + * + * @param pitchmarks + */ + // TODO why is this never used? + public void setPitchmarks(int[] pitchmarks) { + this.pitchmarks = pitchmarks; + } + + public int[] getPitchmarks() { + return pitchmarks; + } + + /** + * Get the pitchmark marking the end of the period with the index number periodIndex. + * + * @param periodIndex + * @return the pitchmark position, in samples + */ + public int getPitchmark(int periodIndex) { + return pitchmarks[periodIndex]; + } + + /** + * Get the length of the pitch period ending with pitchmark with the index number periodIndex. + * + * @param periodIndex + * @return the period length, in samples + */ + public int getPeriodLength(int periodIndex) { + if (0 <= periodIndex && periodIndex < pitchmarks.length) { + if (periodIndex > 0) { + return pitchmarks[periodIndex] - pitchmarks[periodIndex - 1]; + } else { + return pitchmarks[periodIndex]; + } + } else { + return 0; + } + } + + public int getNumberOfPitchmarks() { + return pitchmarks.length; + } + + public void setFrames(Datagram[] frames) { + this.frames = frames; + } + + public Datagram[] getFrames() { + return frames; + } + + public void setFrame(int frameIndex, Datagram frame) { + this.frames[frameIndex] = frame; + } + + public Datagram getFrame(int frameIndex) { + return frames[frameIndex]; + } + + public void setRightContextFrame(Datagram aRightContextFrame) { + this.rightContextFrame = aRightContextFrame; + } + + public Datagram getRightContextFrame() { + return rightContextFrame; + } + + /** + * Set the realised duration of this unit, in samples. + * + * @param duration + */ + public void setUnitDuration(int duration) { + this.unitDuration = duration; + } + + /** + * Get the realised duration of this unit, in samples + * + * @return + */ + public int getUnitDuration() { + return unitDuration; + } + + } +} diff --git a/marytts-unitselection/src/main/java/marytts/unitselection/concat/DatagramOverlapDoubleDataSource.java b/marytts-unitselection/src/main/java/marytts/unitselection/concat/DatagramOverlapDoubleDataSource.java new file mode 100644 index 0000000000..4e19796697 --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/unitselection/concat/DatagramOverlapDoubleDataSource.java @@ -0,0 +1,148 @@ +/** + * Copyright 2004-2006 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.unitselection.concat; + +import marytts.signalproc.window.DynamicTwoHalvesWindow; +import marytts.signalproc.window.Window; +import marytts.util.data.BufferedDoubleDataSource; +import marytts.util.data.Datagram; +import marytts.util.data.DoubleDataSource; + +public class DatagramOverlapDoubleDataSource extends BufferedDoubleDataSource { + protected Datagram[][] datagrams; + protected Datagram[] rightContexts; + protected int p; // point to current datagrams/rightContext + protected int q; // point to current datagram within datagrams[p] + protected int totalRead; // count samples read from datagrams + + /** + * Construct an double data source from the given array of datagram arrays and right contexts. + * + * @param datagrams + */ + public DatagramOverlapDoubleDataSource(Datagram[][] datagrams, Datagram[] rightContexts) { + super((DoubleDataSource) null); + this.datagrams = datagrams; + this.rightContexts = rightContexts; + dataLength = 0; + for (int i = 0; i < datagrams.length; i++) { + for (int j = 0; j < datagrams[i].length; j++) { + dataLength += datagrams[i][j].getDuration(); + } + } + p = 0; + q = 0; + } + + /** + * Whether or not any more data can be read from this data source. + * + * @return true if another call to getData() will return data, false otherwise. + */ + public boolean hasMoreData() { + if (currentlyInBuffer() > 0 || totalRead < dataLength) + return true; + return false; + } + + /** + * The number of doubles that can currently be read from this double data source without blocking. This number can change over + * time. + * + * @return the number of doubles that can currently be read without blocking + */ + public int available() { + int available = (int) (currentlyInBuffer() + dataLength - totalRead); + return available; + } + + /** + * Attempt to get more data from the input source. If less than this can be read, the possible amount will be read, but + * canReadMore() will return false afterwards. + * + * @param minLength + * the amount of data to get from the input source + * @return true if the requested amount could be read, false if none or less data could be read. + */ + protected boolean readIntoBuffer(int minLength) { + if (bufferSpaceLeft() < minLength) { + // current buffer cannot hold the data requested; + // need to make it larger + increaseBufferSize(minLength + currentlyInBuffer()); + } else if (buf.length - writePos < minLength) { + compact(); // create a contiguous space for the new data + } + // Now we have a buffer that can hold at least minLength new data points + int readSum = 0; + // read blocks: + + while (readSum < minLength && p < datagrams.length) { + if (q >= datagrams[p].length) { + p++; + q = 0; + } else { + Datagram next = datagrams[p][q]; + int length = (int) next.getDuration(); + // System.out.println("Unit duration = " + String.valueOf(length)); + if (buf.length < writePos + length) { + increaseBufferSize(writePos + length); + } + int read = readDatagram(next, buf, writePos); + if (q == 0 && p > 0 && rightContexts[p - 1] != null) { + // overlap-add situation + // window the data that we have just read with the left half of a HANN window: + new DynamicTwoHalvesWindow(Window.HANNING).applyInlineLeftHalf(buf, writePos, read); + // and overlap-add the previous right context, windowed with the right half of a HANN window: + double[] context = new double[(int) rightContexts[p - 1].getDuration()]; + readDatagram(rightContexts[p - 1], context, 0); + new DynamicTwoHalvesWindow(Window.HANNING).applyInlineRightHalf(context, 0, context.length); + for (int i = 0, iMax = Math.min(read, context.length); i < iMax; i++) { + buf[writePos + i] += context[i]; + } + } + writePos += read; + readSum += read; + totalRead += read; + q++; + } + } + if (dataProcessor != null) { + dataProcessor.applyInline(buf, writePos - readSum, readSum); + } + return readSum >= minLength; + } + + protected int readDatagram(Datagram d, double[] target, int pos) { + int dur = (int) d.getDuration(); + byte[] frameAudio = d.getData(); + assert frameAudio.length / 2 == dur : "expected datagram data length to be " + (dur * 2) + ", found " + frameAudio.length; + for (int i = 0; i < frameAudio.length; i += 2, pos++) { + int sample; + byte lobyte; + byte hibyte; + // big endian: + lobyte = frameAudio[i + 1]; + hibyte = frameAudio[i]; + sample = hibyte << 8 | lobyte & 0xFF; + target[pos] = sample / 32768.0;// normalise to range [-1, 1]; + } + return dur; + } +} diff --git a/marytts-unitselection/src/main/java/marytts/unitselection/concat/FdpsolaUnitConcatenator.java b/marytts-unitselection/src/main/java/marytts/unitselection/concat/FdpsolaUnitConcatenator.java new file mode 100644 index 0000000000..1f21b970e0 --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/unitselection/concat/FdpsolaUnitConcatenator.java @@ -0,0 +1,601 @@ +/** + * Copyright 2007 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.unitselection.concat; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import javax.sound.sampled.AudioInputStream; + +import marytts.modules.phonemiser.Allophone; +import marytts.server.MaryProperties; +import marytts.signalproc.process.FDPSOLAProcessor; +import marytts.unitselection.analysis.Phone; +import marytts.unitselection.select.SelectedUnit; +import marytts.features.Target; +import marytts.util.data.Datagram; +import marytts.util.data.audio.DDSAudioInputStream; +import marytts.util.math.MathUtils; + +/** + * A unit concatenator that supports FD-PSOLA based prosody modifications during speech synthesis + * + * @author Oytun Türk, modified by steiner + * + */ +public class FdpsolaUnitConcatenator extends OverlapUnitConcatenator { + + // modification value ranges with hard-coded defaults: + private double minTimeScaleFactor = 0.5; + private double maxTimeScaleFactor = 2.0; + private double minPitchScaleFactor = 0.5; + private double maxPitchScaleFactor = 2.0; + + /** + * + */ + public FdpsolaUnitConcatenator() { + super(); + } + + /** + * Alternative constructor that allows overriding the modification value ranges + * + * @param minTimeScaleFactor + * minimum duration scale factor + * @param maxTimeScaleFactor + * maximum duration scale factor + * @param minPitchScaleFactor + * minimum F0 scale factor + * @param maxPitchScaleFactor + * maximum F0 scale factor + */ + public FdpsolaUnitConcatenator(double minTimeScaleFactor, double maxTimeScaleFactor, double minPitchScaleFactor, + double maxPitchScaleFactor) { + super(); + this.minTimeScaleFactor = minTimeScaleFactor; + this.maxTimeScaleFactor = maxTimeScaleFactor; + this.minPitchScaleFactor = minPitchScaleFactor; + this.maxPitchScaleFactor = maxPitchScaleFactor; + } + + /** + * Get the Datagrams from a List of SelectedUnits as an array of arrays; the number of elements in the array is equal to the + * number of Units, and each element contains that Unit's Datagrams as an array. + * + * @param units + * @return array of Datagram arrays + */ + private Datagram[][] getDatagrams(List units) { + Datagram[][] datagrams = new Datagram[units.size()][]; + for (int i = 0; i < units.size(); i++) { + UnitData unitData = (UnitData) units.get(i).getConcatenationData(); + datagrams[i] = unitData.getFrames(); + } + return datagrams; + } + + /** + * Convenience method to return the rightmost Datagram from each element in a List of SelectedUnits + * + * @param units + * @return rightmost Datagrams as an array + */ + private Datagram[] getRightContexts(List units) { + Datagram[] rightContexts = new Datagram[units.size()]; + for (int i = 0; i < rightContexts.length; i++) { + SelectedUnit unit = units.get(i); + UnitData unitData = (UnitData) unit.getConcatenationData(); + rightContexts[i] = unitData.getRightContextFrame(); + } + return rightContexts; + } + + /** + * Get voicing for every Datagram in a List of SelectedUnits, as an array of arrays of booleans. This queries the phonological + * voicedness value for the Target as defined in the AllophoneSet + * + * @param units + * @return array of boolean voicing arrays + */ + private boolean[][] getVoicings(List units) { + Datagram[][] datagrams = getDatagrams(units); + + boolean[][] voicings = new boolean[datagrams.length][]; + + for (int i = 0; i < datagrams.length; i++) { + Allophone allophone = units.get(i).getTarget().getAllophone(); + + voicings[i] = new boolean[datagrams[i].length]; + + if (allophone != null && allophone.isVoiced()) { + Arrays.fill(voicings[i], true); + } else { + Arrays.fill(voicings[i], false); + } + } + return voicings; + } + + // We can try different things in this function + // 1) Pitch of the selected units can be smoothed without using the target pitch values at all. + // This will involve creating the target f0 values for each frame by ensuing small adjustments and yet reduce pitch + // discontinuity + // 2) Pitch of the selected units can be modified to match the specified target where those target values are smoothed + // 3) A mixture of (1) and (2) can be devised, i.e. to minimize the amount of pitch modification one of the two methods can be + // selected for a given unit + // 4) Pitch segments of selected units can be shifted + // 5) Pitch segments of target units can be shifted + // 6) Pitch slopes can be modified for better matching in concatenation boundaries + private double[][] getPitchScales(List units) { + Datagram[][] datagrams = getDatagrams(units); + int len = datagrams.length; + int i, j; + double averageUnitF0InHz; + double averageTargetF0InHz; + int totalTargetUnits; + double[][] pscales = new double[len][]; + SelectedUnit prevUnit = null; + SelectedUnit unit = null; + SelectedUnit nextUnit = null; + + Target prevTarget = null; + Target target = null; + Target nextTarget = null; + + // Estimation of pitch scale modification amounts + for (i = 0; i < len; i++) { + if (i > 0) + prevUnit = (SelectedUnit) units.get(i - 1); + else + prevUnit = null; + + unit = (SelectedUnit) units.get(i); + + if (i < len - 1) + nextUnit = (SelectedUnit) units.get(i + 1); + else + nextUnit = null; + + // get Targets for these three Units: + if (prevUnit != null) { + prevTarget = prevUnit.getTarget(); + } + target = unit.getTarget(); + if (nextUnit != null) { + nextTarget = nextUnit.getTarget(); + } + + Allophone allophone = unit.getTarget().getAllophone(); + + int totalDatagrams = 0; + averageUnitF0InHz = 0.0; + averageTargetF0InHz = 0.0; + totalTargetUnits = 0; + + // so we are getting the mean F0 for each unit over a 3-unit window?? + // don't process previous Target if it's null or silence: + if (i > 0 && prevTarget != null && !prevTarget.isSilence()) { + for (j = 0; j < datagrams[i - 1].length; j++) { + // why not use voicings? + if (allophone != null && (allophone.isVowel() || allophone.isVoiced())) { + averageUnitF0InHz += ((double) timeline.getSampleRate()) / ((double) datagrams[i - 1][j].getDuration()); + totalDatagrams++; + } + } + + averageTargetF0InHz += prevTarget.getTargetF0InHz(); + totalTargetUnits++; + } + + // don't process Target if it's null or silence: + if (target != null && !target.isSilence()) { + for (j = 0; j < datagrams[i].length; j++) { + if (allophone != null && (allophone.isVowel() || allophone.isVoiced())) { + averageUnitF0InHz += ((double) timeline.getSampleRate()) / ((double) datagrams[i][j].getDuration()); + totalDatagrams++; + } + + averageTargetF0InHz += target.getTargetF0InHz(); + totalTargetUnits++; + } + } + + // don't process next Target if it's null or silence: + if (i < len - 1 && prevTarget != null && !prevTarget.isSilence()) { + for (j = 0; j < datagrams[i + 1].length; j++) { + if (allophone != null && (allophone.isVowel() || allophone.isVoiced())) { + averageUnitF0InHz += ((double) timeline.getSampleRate()) / ((double) datagrams[i + 1][j].getDuration()); + totalDatagrams++; + } + } + + averageTargetF0InHz += nextTarget.getTargetF0InHz(); + totalTargetUnits++; + } + + averageTargetF0InHz /= totalTargetUnits; + averageUnitF0InHz /= totalDatagrams; + // so what was all that for?? these average frequencies are never used... + + pscales[i] = new double[datagrams[i].length]; + + for (j = 0; j < datagrams[i].length; j++) { + if (allophone != null && allophone.isVoiced()) { + /* + * pscales[i][j] = averageTargetF0InHz/averageUnitF0InHz; if (pscales[i][j]>1.2) pscales[i][j]=1.2; if + * (pscales[i][j]<0.8) pscales[i][j]=0.8; + */ + pscales[i][j] = 1.0; + } else { + pscales[i][j] = 1.0; + } + } + } + return pscales; + } + + // We can try different things in this function + // 1) Duration modification factors can be estimated using neighbouring selected and target unit durations + // 2) Duration modification factors can be limited or even set to 1.0 for different phone classes + // 3) Duration modification factors can be limited depending on the previous/next phone class + private double[][] getDurationScales(List units) { + Datagram[][] datagrams = getDatagrams(units); + int len = datagrams.length; + + int i, j; + double[][] tscales = new double[len][]; + int unitDuration; + + double[] unitDurationsInSeconds = new double[datagrams.length]; + + SelectedUnit prevUnit = null; + SelectedUnit unit = null; + SelectedUnit nextUnit = null; + + for (i = 0; i < len; i++) { + unitDuration = 0; + for (j = 0; j < datagrams[i].length; j++) { + if (j == datagrams[i].length - 1) { + // if (rightContexts!=null && rightContexts[i]!=null) + // unitDuration += datagrams[i][j].getDuration();//+rightContexts[i].getDuration(); + // else + unitDuration += datagrams[i][j].getDuration(); + } else + unitDuration += datagrams[i][j].getDuration(); + } + unitDurationsInSeconds[i] = ((double) unitDuration) / timeline.getSampleRate(); + } + + double targetDur, unitDur; + for (i = 0; i < len; i++) { + targetDur = 0.0; + unitDur = 0.0; + // commented out dead code: + // if (false && i>0) + // { + // prevUnit = (SelectedUnit) units.get(i-1); + // targetDur += prevUnit.getTarget().getTargetDurationInSeconds(); + // unitDur += unitDurationsInSeconds[i-1]; + // } + + unit = (SelectedUnit) units.get(i); + targetDur += unit.getTarget().getTargetDurationInSeconds(); + unitDur += unitDurationsInSeconds[i]; + + // commented out dead code: + // if (false && i1.2) + // tscales[i][j]=1.2; + // if (tscales[i][j]<0.8) + // tscales[i][j]=0.8; + + // tscales[i][j] = 1.2; + } + logger.debug("time scaling factor for unit " + unit.getTarget().getName() + " -> " + targetDur / unitDur); + } + return tscales; + } + + // private double[][] getSyllableBasedPitchScales(List units) { + // List phones = ProsodyAnalyzer.parseIntoPhones(units, timeline.getSampleRate()); + // List syllables = Syllable.parseIntoSyllables(phones); + // ListIterator syllableIterator = syllables.listIterator(); + // while (syllableIterator.hasNext()) { + // if (!syllableIterator.hasPrevious()) { + // continue; + // } + // // TODO unfinished! + // } + // return null; + // } + + private double[][] getPhoneBasedDurationScales(List units) { + + List timeScaleFactors = prosodyAnalyzer.getDurationFactors(); + + // finally, initialize the tscales array... + double[][] tscales = new double[timeScaleFactors.size()][]; + Datagram[][] datagrams = getDatagrams(units); + for (int i = 0; i < tscales.length; i++) { + tscales[i] = new double[datagrams[i].length]; + // ...which currently provides the same time scale factor for every datagram in a selected unit: + Arrays.fill(tscales[i], timeScaleFactors.get(i)); + } + + // for quick and dirty debugging, dump tscales to Praat DurationTier: + try { + prosodyAnalyzer.writePraatDurationTier(MaryProperties.maryBase() + "/tscales.DurationTier"); + } catch (IOException e) { + logger.warn("Could not dump tscales to file"); + } + + return tscales; + } + + /** + * Convenience method to grep those SelectedUnits from a List which have positive duration + * + * @param units + * @return units with positive duration + */ + @Deprecated + private List getNonEmptyUnits(List units) { + ArrayList nonEmptyUnits = new ArrayList(units.size()); + for (SelectedUnit unit : units) { + UnitData unitData = (UnitData) unit.getConcatenationData(); + if (unitData.getUnitDuration() > 0 && unit.getTarget().getMaryxmlElement() != null) { + nonEmptyUnits.add(unit); + } + } + return nonEmptyUnits; + } + + protected Datagram[][] getRealizedDatagrams(List phones) { + List datagramList = new ArrayList(); + for (Phone phone : phones) { + if (phone.getLeftTargetDuration() > 0) { + Datagram[] leftDatagrams = phone.getLeftUnitFrames(); + datagramList.add(leftDatagrams); + } + if (phone.getRightTargetDuration() > 0) { + Datagram[] rightDatagrams = phone.getRightUnitFrames(); + datagramList.add(rightDatagrams); + } + } + Datagram[][] datagramArray = datagramList.toArray(new Datagram[datagramList.size()][]); + return datagramArray; + } + + protected Datagram[] getRealizedRightContexts(List phones) { + List datagramList = new ArrayList(); + for (Phone phone : phones) { + if (phone.getLeftTargetDuration() > 0) { + UnitData leftUnitData = phone.getLeftUnitData(); + Datagram leftRightContext = leftUnitData.getRightContextFrame(); + datagramList.add(leftRightContext); + } + if (phone.getRightTargetDuration() > 0) { + UnitData rightUnitData = phone.getRightUnitData(); + Datagram rightRightContext = rightUnitData.getRightContextFrame(); + datagramList.add(rightRightContext); + } + } + Datagram[] datagramArray = datagramList.toArray(new Datagram[datagramList.size()]); + return datagramArray; + } + + private boolean[][] getRealizedVoicings(List phones) { + List voicingList = new ArrayList(); + for (Phone phone : phones) { + boolean voiced = phone.isVoiced(); + if (phone.getLeftTargetDuration() > 0) { + int leftNumberOfFrames = phone.getNumberOfLeftUnitFrames(); + boolean[] leftVoiceds = new boolean[leftNumberOfFrames]; + Arrays.fill(leftVoiceds, voiced); + voicingList.add(leftVoiceds); + } + if (phone.getRightTargetDuration() > 0) { + int rightNumberOfFrames = phone.getNumberOfRightUnitFrames(); + boolean[] rightVoiceds = new boolean[rightNumberOfFrames]; + Arrays.fill(rightVoiceds, voiced); + voicingList.add(rightVoiceds); + } + } + boolean[][] voicingArray = voicingList.toArray(new boolean[voicingList.size()][]); + return voicingArray; + } + + private double[][] getRealizedTimeScales(List phones) { + List durationFactorList = new ArrayList(phones.size()); + for (Phone phone : phones) { + if (phone.getLeftTargetDuration() > 0) { + int leftNumberOfFrames = phone.getNumberOfLeftUnitFrames(); + double leftDurationFactor = phone.getLeftDurationFactor(); + // scale the factor to reasonably safe values: + if (leftDurationFactor < minTimeScaleFactor) { + String message = "Left duration factor (" + leftDurationFactor + ") for phone " + phone + " too small;"; + leftDurationFactor = minTimeScaleFactor; + message += " clipped to " + leftDurationFactor; + logger.debug(message); + } else if (leftDurationFactor > maxTimeScaleFactor) { + String message = "Left duration factor (" + leftDurationFactor + ") for phone " + phone + " too large;"; + leftDurationFactor = maxTimeScaleFactor; + message += " clipped to " + leftDurationFactor; + logger.debug(message); + } + double[] leftDurationFactors = new double[leftNumberOfFrames]; + Arrays.fill(leftDurationFactors, leftDurationFactor); + durationFactorList.add(leftDurationFactors); + } + if (phone.getRightTargetDuration() > 0) { + int rightNumberOfFrames = phone.getNumberOfRightUnitFrames(); + double rightDurationFactor = phone.getRightDurationFactor(); + if (phone.isTransient()) { + rightDurationFactor = 1; // never modify the duration of a burst + } + // scale the factor to reasonably safe values: + if (rightDurationFactor < minTimeScaleFactor) { + String message = "Right duration factor (" + rightDurationFactor + ") for phone " + phone + " too small;"; + rightDurationFactor = minTimeScaleFactor; + message += " clipped to " + rightDurationFactor; + logger.debug(message); + } else if (rightDurationFactor > maxTimeScaleFactor) { + String message = "Right duration factor (" + rightDurationFactor + ") for phone " + phone + " too large;"; + rightDurationFactor = maxTimeScaleFactor; + message += " clipped to " + rightDurationFactor; + logger.debug(message); + } + double[] rightDurationFactors = new double[rightNumberOfFrames]; + Arrays.fill(rightDurationFactors, rightDurationFactor); + durationFactorList.add(rightDurationFactors); + } + } + double[][] durationFactorArray = durationFactorList.toArray(new double[durationFactorList.size()][]); + return durationFactorArray; + } + + private double[][] getRealizedPitchScales(List phones) { + List f0FactorList = new ArrayList(phones.size()); + for (Phone phone : phones) { + if (phone.getLeftTargetDuration() > 0) { + int leftNumberOfFrames = phone.getNumberOfLeftUnitFrames(); + double[] leftF0Factors = phone.getLeftF0Factors(); + boolean clipped = MathUtils.clipRange(leftF0Factors, minPitchScaleFactor, maxPitchScaleFactor); + if (clipped) { + logger.debug("Left F0 factors for phone " + phone + " contained out-of-range values; clipped to [" + + minPitchScaleFactor + ", " + maxPitchScaleFactor + "]"); + } + f0FactorList.add(leftF0Factors); + } + if (phone.getRightTargetDuration() > 0) { + int rightNumberOfFrames = phone.getNumberOfRightUnitFrames(); + double[] rightF0Factors = phone.getRightF0Factors(); + boolean clipped = MathUtils.clipRange(rightF0Factors, minPitchScaleFactor, maxPitchScaleFactor); + if (clipped) { + logger.debug("Left F0 factors for phone " + phone + " contained out-of-range values; clipped to [" + + minPitchScaleFactor + ", " + maxPitchScaleFactor + "]"); + } + f0FactorList.add(rightF0Factors); + } + } + double[][] f0FactorArray = f0FactorList.toArray(new double[f0FactorList.size()][]); + return f0FactorArray; + } + + /** + * Generate audio to match the target pitchmarks as closely as possible. + * + * @param units + * @return + * @throws IOException + */ + protected AudioInputStream generateAudioStream(List units) throws IOException { + // gather arguments for FDPSOLA processing: + // Datagram[][] datagrams = getDatagrams(units); + // Datagram[] rightContexts = getRightContexts(units); + // boolean[][] voicings = getVoicings(units); + // double[][] pscales = getPitchScales(units); + // double[][] tscales = getDurationScales(units); + // double[][] tscales = getPhoneBasedDurationScales(units); + + List realizedPhones = prosodyAnalyzer.getRealizedPhones(); + Datagram[][] datagrams = getRealizedDatagrams(realizedPhones); + Datagram[] rightContexts = getRealizedRightContexts(realizedPhones); + boolean[][] voicings = getRealizedVoicings(realizedPhones); + double[][] tscales = getRealizedTimeScales(realizedPhones); + double[][] pscales = getRealizedPitchScales(realizedPhones); + + // process into audio stream: + DDSAudioInputStream stream = (new FDPSOLAProcessor()).processDecrufted(datagrams, rightContexts, audioformat, voicings, + pscales, tscales); + + // update durations from processed Datagrams: + // updateUnitDataDurations(units, datagrams); + updateRealizedUnitDataDurations(realizedPhones, datagrams); + + return stream; + } + + /** + * Explicitly propagate durations of Datagrams to UnitData for each SelectedUnit; those durations are otherwise oblivious to + * the data they describe... + * + * @param units + * whose data should have its durations updated + * @param datagrams + * processed array of arrays of Datagrams which had their durations updated in + * {@link FDPSOLAProcessor#processDecrufted} + */ + private void updateUnitDataDurations(List units, Datagram[][] datagrams) { + for (int i = 0; i < datagrams.length; i++) { + SelectedUnit unit = units.get(i); + UnitData unitData = (UnitData) unit.getConcatenationData(); + int unitDuration = 0; + for (int j = 0; j < datagrams[i].length; j++) { + int datagramDuration = (int) datagrams[i][j].getDuration(); + unitData.getFrame(j).setDuration(datagramDuration); + unitDuration += datagramDuration; + } + unitData.setUnitDuration(unitDuration); + } + } + + private void updateRealizedUnitDataDurations(List phones, Datagram[][] datagrams) { + int phIndex = 0; + for (Phone phone : phones) { + if (phone.getLeftTargetDuration() > 0) { + UnitData leftUnitData = phone.getLeftUnitData(); + int leftUnitDataDuration = 0; + for (int dg = 0; dg < datagrams[phIndex].length; dg++) { + int datagramDuration = (int) datagrams[phIndex][dg].getDuration(); + leftUnitData.getFrame(dg).setDuration(datagramDuration); + leftUnitDataDuration += datagramDuration; + } + phIndex++; + leftUnitData.setUnitDuration(leftUnitDataDuration); + } + if (phone.getRightTargetDuration() > 0) { + UnitData rightUnitData = phone.getRightUnitData(); + int rightUnitDataDuration = 0; + for (int dg = 0; dg < datagrams[phIndex].length; dg++) { + int datagramDuration = (int) datagrams[phIndex][dg].getDuration(); + rightUnitData.getFrame(dg).setDuration(datagramDuration); + rightUnitDataDuration += datagramDuration; + } + phIndex++; + rightUnitData.setUnitDuration(rightUnitDataDuration); + } + } + } +} diff --git a/marytts-unitselection/src/main/java/marytts/unitselection/concat/HnmUnitConcatenator.java b/marytts-unitselection/src/main/java/marytts/unitselection/concat/HnmUnitConcatenator.java new file mode 100644 index 0000000000..fe4d78363a --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/unitselection/concat/HnmUnitConcatenator.java @@ -0,0 +1,251 @@ +/** + * Copyright 2007 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * Permission is hereby granted, free of charge, to use and distribute + * this software and its documentation without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of this work, and to + * permit persons to whom this work is furnished to do so, subject to + * the following conditions: + * + * 1. The code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * 2. Any modifications must be clearly marked as such. + * 3. Original authors' names are not deleted. + * 4. The authors' names are not used to endorse or promote products + * derived from this software without specific prior written + * permission. + * + * DFKI GMBH AND THE CONTRIBUTORS TO THIS WORK DISCLAIM ALL WARRANTIES WITH + * REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL DFKI GMBH NOR THE + * CONTRIBUTORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL + * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR + * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS + * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF + * THIS SOFTWARE. + */ + +package marytts.unitselection.concat; + +import java.io.IOException; +import java.util.List; + +import javax.sound.sampled.AudioInputStream; + +import marytts.signalproc.adaptation.prosody.BasicProsodyModifierParams; +import marytts.signalproc.sinusoidal.hntm.analysis.HntmAnalyzerParams; +import marytts.signalproc.sinusoidal.hntm.analysis.HntmSpeechFrame; +import marytts.signalproc.sinusoidal.hntm.analysis.HntmSpeechSignal; +import marytts.signalproc.sinusoidal.hntm.synthesis.HntmSynthesizedSignal; +import marytts.signalproc.sinusoidal.hntm.synthesis.HntmSynthesizer; +import marytts.signalproc.sinusoidal.hntm.synthesis.HntmSynthesizerParams; +import marytts.unitselection.data.HnmDatagram; +import marytts.unitselection.data.Unit; +import marytts.unitselection.select.SelectedUnit; +import marytts.util.data.BufferedDoubleDataSource; +import marytts.util.data.Datagram; +import marytts.util.data.audio.DDSAudioInputStream; +import marytts.util.math.MathUtils; + +/** + * A unit concatenator for harmonics plus noise based speech synthesis + * + * @author Oytun Türk + * + */ +public class HnmUnitConcatenator extends OverlapUnitConcatenator { + + public HnmUnitConcatenator() { + super(); + } + + /** + * Get the raw audio material for each unit from the timeline. + * + * @param units + */ + protected void getDatagramsFromTimeline(List units) throws IOException { + for (SelectedUnit unit : units) { + assert !unit.getUnit().isEdgeUnit() : "We should never have selected any edge units!"; + HnmUnitData unitData = new HnmUnitData(); + unit.setConcatenationData(unitData); + int nSamples = 0; + int unitSize = unitToTimeline(unit.getUnit().duration); // convert to timeline samples + long unitStart = unitToTimeline(unit.getUnit().startTime); // convert to timeline samples + // System.out.println("Unit size "+unitSize+", pitchmarksInUnit "+pitchmarksInUnit); + // System.out.println(unitStart/((float)timeline.getSampleRate())); + // System.out.println("Unit index = " + unit.getUnit().getIndex()); + + Datagram[] datagrams = timeline.getDatagrams(unitStart, (long) unitSize); + unitData.setFrames(datagrams); + + // one left context period for windowing: + Datagram leftContextFrame = null; + Unit prevInDB = database.getUnitFileReader().getPreviousUnit(unit.getUnit()); + long unitPrevStart = unitToTimeline(prevInDB.startTime); // convert to timeline samples + if (prevInDB != null && !prevInDB.isEdgeUnit()) { + long unitPrevSize = unitToTimeline(prevInDB.duration); + Datagram[] unitPrevDatagrams = timeline.getDatagrams(unitPrevStart, (long) unitPrevSize); + // leftContextFrame = timeline.getDatagram(unitPrevStart); + if (unitPrevDatagrams != null && unitPrevDatagrams.length > 0) { + leftContextFrame = unitPrevDatagrams[unitPrevDatagrams.length - 1]; + } + unitData.setLeftContextFrame(leftContextFrame); + } + + // one right context period for windowing: + Datagram rightContextFrame = null; + Unit nextInDB = database.getUnitFileReader().getNextUnit(unit.getUnit()); + if (nextInDB != null && !nextInDB.isEdgeUnit()) { + rightContextFrame = timeline.getDatagram(unitStart + unitSize); + unitData.setRightContextFrame(rightContextFrame); + } + } + } + + /** + * Generate audio to match the target pitchmarks as closely as possible. + * + * @param units + * @return + */ + protected AudioInputStream generateAudioStream(List units) { + int len = units.size(); + Datagram[][] datagrams = new Datagram[len][]; + Datagram[] leftContexts = new Datagram[len]; + Datagram[] rightContexts = new Datagram[len]; + for (int i = 0; i < len; i++) { + SelectedUnit unit = units.get(i); + HnmUnitData unitData = (HnmUnitData) unit.getConcatenationData(); + assert unitData != null : "Should not have null unitdata here"; + Datagram[] frames = unitData.getFrames(); + assert frames != null : "Cannot generate audio from null frames"; + // Generate audio from frames + datagrams[i] = frames; + + Unit prevInDB = database.getUnitFileReader().getPreviousUnit(unit.getUnit()); + Unit prevSelected; + if (i == 0) + prevSelected = null; + else + prevSelected = units.get(i - 1).getUnit(); + if (prevInDB != null && !prevInDB.equals(prevSelected)) { + // Only use left context if we have a previous unit in the DB is not the + // same as the previous selected unit. + leftContexts[i] = (HnmDatagram) unitData.getLeftContextFrame(); // may be null + } + + Unit nextInDB = database.getUnitFileReader().getNextUnit(unit.getUnit()); + Unit nextSelected; + if (i + 1 == len) + nextSelected = null; + else + nextSelected = units.get(i + 1).getUnit(); + if (nextInDB != null && !nextInDB.equals(nextSelected)) { + // Only use right context if we have a next unit in the DB is not the + // same as the next selected unit. + rightContexts[i] = unitData.getRightContextFrame(); // may be null + } + } + + BufferedDoubleDataSource audioSource = synthesize(datagrams, leftContexts, rightContexts); + return new DDSAudioInputStream(audioSource, audioformat); + } + + protected BufferedDoubleDataSource synthesize(Datagram[][] datagrams, Datagram[] leftContexts, Datagram[] rightContexts) { + HntmSynthesizer s = new HntmSynthesizer(); + // TO DO: These should come from timeline and user choices... + HntmAnalyzerParams analysisParams = new HntmAnalyzerParams(); + HntmSynthesizerParams synthesisParams = new HntmSynthesizerParams(); + BasicProsodyModifierParams pmodParams = new BasicProsodyModifierParams(); + int samplingRateInHz = 16000; + + int totalFrm = 0; + int i, j; + float originalDurationInSeconds = 0.0f; + float deltaTimeInSeconds; + long length; + + for (i = 0; i < datagrams.length; i++) { + for (j = 0; j < datagrams[i].length; j++) { + if (datagrams[i][j] != null && (datagrams[i][j] instanceof HnmDatagram)) { + totalFrm++; + length = ((HnmDatagram) datagrams[i][j]).getDuration(); + // deltaTimeInSeconds = SignalProcUtils.sample2time(length, samplingRateInHz); + deltaTimeInSeconds = ((HntmSpeechFrame) ((HnmDatagram) datagrams[i][j]).getFrame()).deltaAnalysisTimeInSeconds; + originalDurationInSeconds += deltaTimeInSeconds; + + // System.out.println("Unit duration = " + String.valueOf(length)); + } + } + } + + HntmSpeechSignal hnmSignal = null; + hnmSignal = new HntmSpeechSignal(totalFrm, samplingRateInHz, originalDurationInSeconds); + HntmSpeechFrame[] leftContextFrames = new HntmSpeechFrame[totalFrm]; + HntmSpeechFrame[] rightContextFrames = new HntmSpeechFrame[totalFrm]; + // + + int frameCount = 0; + float tAnalysisInSeconds = 0.0f; + for (i = 0; i < datagrams.length; i++) { + for (j = 0; j < datagrams[i].length; j++) { + if (datagrams[i][j] != null && (datagrams[i][j] instanceof HnmDatagram) && frameCount < totalFrm) { + tAnalysisInSeconds += ((HntmSpeechFrame) ((HnmDatagram) datagrams[i][j]).getFrame()).deltaAnalysisTimeInSeconds; + + hnmSignal.frames[frameCount] = ((HnmDatagram) datagrams[i][j]).getFrame(); + hnmSignal.frames[frameCount].tAnalysisInSeconds = tAnalysisInSeconds; + + // tAnalysisInSeconds += SignalProcUtils.sample2time(((HnmDatagram)datagrams[i][j]).getDuration(), + // samplingRateInHz); + + if (j == 0) { + if (leftContexts[i] != null && (leftContexts[i] instanceof HnmDatagram)) + leftContextFrames[frameCount] = ((HnmDatagram) leftContexts[i]).getFrame(); + } else { + if (datagrams[i][j - 1] != null && (datagrams[i][j - 1] instanceof HnmDatagram)) + leftContextFrames[frameCount] = ((HnmDatagram) datagrams[i][j - 1]).getFrame(); + } + + if (j == datagrams[i].length - 1) { + if (rightContexts[i] != null && (rightContexts[i] instanceof HnmDatagram)) + rightContextFrames[frameCount] = ((HnmDatagram) rightContexts[i]).getFrame(); + } else { + if (datagrams[i][j + 1] != null && (datagrams[i][j + 1] instanceof HnmDatagram)) + rightContextFrames[frameCount] = ((HnmDatagram) datagrams[i][j + 1]).getFrame(); + } + + frameCount++; + } + } + } + + HntmSynthesizedSignal ss = null; + if (totalFrm > 0) { + ss = s.synthesize(hnmSignal, leftContextFrames, rightContextFrames, pmodParams, null, analysisParams, synthesisParams); + // FileUtils.writeTextFile(hnmSignal.getAnalysisTimes(), "d:\\hnmAnalysisTimes1.txt"); + // FileUtils.writeTextFile(ss.output, "d:\\output.txt"); + if (ss.output != null) + ss.output = MathUtils.multiply(ss.output, 1.0 / 32768.0); + } + + if (ss != null && ss.output != null) + return new BufferedDoubleDataSource(ss.output); + else + return null; + } + + public static class HnmUnitData extends OverlapUnitConcatenator.OverlapUnitData { + protected Datagram leftContextFrame; + + public void setLeftContextFrame(Datagram aLeftContextFrame) { + this.leftContextFrame = aLeftContextFrame; + } + + public Datagram getLeftContextFrame() { + return leftContextFrame; + } + } +} diff --git a/marytts-unitselection/src/main/java/marytts/unitselection/concat/OverlapUnitConcatenator.java b/marytts-unitselection/src/main/java/marytts/unitselection/concat/OverlapUnitConcatenator.java new file mode 100644 index 0000000000..67022d9259 --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/unitselection/concat/OverlapUnitConcatenator.java @@ -0,0 +1,206 @@ +/** + * Copyright 2000-2009 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.unitselection.concat; + +import java.io.IOException; +import java.util.List; + +import javax.sound.sampled.AudioInputStream; + +import marytts.unitselection.data.Unit; +import marytts.unitselection.select.SelectedUnit; +import marytts.util.data.BufferedDoubleDataSource; +import marytts.util.data.Datagram; +import marytts.util.data.DoubleDataSource; +import marytts.util.data.audio.DDSAudioInputStream; + +public class OverlapUnitConcatenator extends BaseUnitConcatenator { + + public OverlapUnitConcatenator() { + super(); + } + + /** + * Get the raw audio material for each unit from the timeline. + * + * @param units + */ + protected void getDatagramsFromTimeline(List units) throws IOException { + for (SelectedUnit unit : units) { + assert !unit.getUnit().isEdgeUnit() : "We should never have selected any edge units!"; + OverlapUnitData unitData = new OverlapUnitData(); + unit.setConcatenationData(unitData); + int nSamples = 0; + int unitSize = unitToTimeline(unit.getUnit().duration); // convert to timeline samples + long unitStart = unitToTimeline(unit.getUnit().startTime); // convert to timeline samples + // System.out.println("Unit size "+unitSize+", pitchmarksInUnit "+pitchmarksInUnit); + // System.out.println(unitStart/((float)timeline.getSampleRate())); + // System.out.println("Unit index = " + unit.getUnit().getIndex()); + + Datagram[] datagrams = timeline.getDatagrams(unitStart, (long) unitSize); + unitData.setFrames(datagrams); + // one right context period for windowing: + Datagram rightContextFrame = null; + Unit nextInDB = database.getUnitFileReader().getNextUnit(unit.getUnit()); + if (nextInDB != null && !nextInDB.isEdgeUnit()) { + rightContextFrame = timeline.getDatagram(unitStart + unitSize); + unitData.setRightContextFrame(rightContextFrame); + } + } + } + + /** + * Determine target pitchmarks (= duration and f0) for each unit. + * + * @param units + */ + protected void determineTargetPitchmarks(List units) { + for (SelectedUnit unit : units) { + UnitData unitData = (UnitData) unit.getConcatenationData(); + assert unitData != null : "Should not have null unitdata here"; + Datagram[] datagrams = unitData.getFrames(); + Datagram[] frames = null; // frames to realise + // The number and duration of the frames to realise + // must be the result of the target pitchmark computation. + + // Set target pitchmarks, + // either by copying from units (data-driven) + // or by computing from target (model-driven) + int unitDuration = 0; + int nZeroLengthDatagrams = 0; + for (int i = 0; i < datagrams.length; i++) { + int dur = (int) datagrams[i].getDuration(); + if (dur == 0) + nZeroLengthDatagrams++; + unitDuration += datagrams[i].getDuration(); + } + if (nZeroLengthDatagrams > 0) { + logger.warn("Unit " + unit + " contains " + nZeroLengthDatagrams + " zero-length datagrams -- removing them"); + Datagram[] dummy = new Datagram[datagrams.length - nZeroLengthDatagrams]; + for (int i = 0, j = 0; i < datagrams.length; i++) { + if (datagrams[i].getDuration() > 0) { + dummy[j++] = datagrams[i]; + } + } + datagrams = dummy; + unitData.setFrames(datagrams); + } + if (unit.getTarget().isSilence()) { + int targetDuration = Math.round(unit.getTarget().getTargetDurationInSeconds() * audioformat.getSampleRate()); + if (targetDuration > 0 && datagrams != null && datagrams.length > 0) { + int firstPeriodDur = (int) datagrams[0].getDuration(); + if (targetDuration < firstPeriodDur) { + logger.debug("For " + unit + ", adjusting target duration to be at least one period: " + + (firstPeriodDur / audioformat.getSampleRate()) + " s instead of requested " + + unit.getTarget().getTargetDurationInSeconds() + " s"); + targetDuration = firstPeriodDur; + } + if (unitDuration < targetDuration) { + // insert silence in the middle + frames = new Datagram[datagrams.length + 1]; + int mid = (datagrams.length + 1) / 2; + System.arraycopy(datagrams, 0, frames, 0, mid); + if (mid < datagrams.length) { + System.arraycopy(datagrams, mid, frames, mid + 1, datagrams.length - mid); + } + frames[mid] = createZeroDatagram(targetDuration - unitDuration); + } else { // unitDuration >= targetDuration + // cut frames from the middle + int midright = (datagrams.length + 1) / 2; // first frame of the right part + int midleft = midright - 1; // last frame of the left part + while (unitDuration > targetDuration && midright < datagrams.length) { + unitDuration -= datagrams[midright].getDuration(); + midright++; + if (unitDuration > targetDuration && midleft > 0) { // force it to leave at least one frame, therefore + // > 0 + unitDuration -= datagrams[midleft].getDuration(); + midleft--; + } + } + frames = new Datagram[midleft + 1 + datagrams.length - midright]; + assert midleft >= 0; + System.arraycopy(datagrams, 0, frames, 0, midleft + 1); + if (midright < datagrams.length) { + System.arraycopy(datagrams, midright, frames, midleft + 1, datagrams.length - midright); + } + } + unitDuration = targetDuration; // now they are the same + } else { // unitSize == 0, we have a zero-length silence unit + // artificial silence data: + frames = new Datagram[] { createZeroDatagram(targetDuration) }; + unitDuration = targetDuration; + } + } else { // not silence + // take unit as is + frames = datagrams; + } + unitData.setUnitDuration(unitDuration); + unitData.setFrames(frames); + } + } + + /** + * Generate audio to match the target pitchmarks as closely as possible. + * + * @param units + * @return + * @throws IOException + */ + protected AudioInputStream generateAudioStream(List units) throws IOException { + int len = units.size(); + Datagram[][] datagrams = new Datagram[len][]; + Datagram[] rightContexts = new Datagram[len]; + for (int i = 0; i < len; i++) { + SelectedUnit unit = units.get(i); + OverlapUnitData unitData = (OverlapUnitData) unit.getConcatenationData(); + assert unitData != null : "Should not have null unitdata here"; + Datagram[] frames = unitData.getFrames(); + assert frames != null : "Cannot generate audio from null frames"; + // Generate audio from frames + datagrams[i] = frames; + Unit nextInDB = database.getUnitFileReader().getNextUnit(unit.getUnit()); + Unit nextSelected; + if (i + 1 == len) + nextSelected = null; + else + nextSelected = units.get(i + 1).getUnit(); + if (nextInDB != null && !nextInDB.equals(nextSelected)) { + // Only use right context if we have a next unit in the DB is not the + // same as the next selected unit. + rightContexts[i] = unitData.getRightContextFrame(); // may be null + } + } + + DoubleDataSource audioSource = new DatagramOverlapDoubleDataSource(datagrams, rightContexts); + return new DDSAudioInputStream(new BufferedDoubleDataSource(audioSource), audioformat); + } + + public static class OverlapUnitData extends BaseUnitConcatenator.UnitData { + protected Datagram rightContextFrame; + + public void setRightContextFrame(Datagram aRightContextFrame) { + this.rightContextFrame = aRightContextFrame; + } + + public Datagram getRightContextFrame() { + return rightContextFrame; + } + } +} diff --git a/marytts-unitselection/src/main/java/marytts/unitselection/concat/UnitConcatenator.java b/marytts-unitselection/src/main/java/marytts/unitselection/concat/UnitConcatenator.java new file mode 100644 index 0000000000..5232ba8871 --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/unitselection/concat/UnitConcatenator.java @@ -0,0 +1,61 @@ +/** + * Copyright 2006 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.unitselection.concat; + +import java.io.IOException; +import java.util.List; + +import javax.sound.sampled.AudioFormat; +import javax.sound.sampled.AudioInputStream; + +import marytts.unitselection.data.UnitDatabase; +import marytts.unitselection.select.SelectedUnit; + +/** + * Concatenates the units of an utterance and returns an audio stream + * + * @author Marc Schröder, Anna Hunecke + * + */ +public interface UnitConcatenator { + /** + * Initialise the unit concatenator from the database. + * + * @param database + */ + public void load(UnitDatabase database); + + /** + * Build the audio stream from the units + * + * @param units + * the units + * @return the resulting audio stream + */ + public AudioInputStream getAudio(List units) throws IOException; + + /** + * Provide the audio format which will be produced by this unit concatenator. + * + * @return the audio format + */ + public AudioFormat getAudioFormat(); + +} diff --git a/marytts-unitselection/src/main/java/marytts/unitselection/data/DiphoneUnit.java b/marytts-unitselection/src/main/java/marytts/unitselection/data/DiphoneUnit.java new file mode 100644 index 0000000000..bc57790ec9 --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/unitselection/data/DiphoneUnit.java @@ -0,0 +1,65 @@ +/** + * Copyright 2006 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.unitselection.data; + +public class DiphoneUnit extends Unit { + public final Unit left; + public final Unit right; + + public DiphoneUnit(Unit left, Unit right) { + super(left.startTime, left.duration + right.duration, left.index); + this.left = left; + this.right = right; + } + + public int getIndex() { + throw new IllegalStateException("This method should not be called for DiphoneUnits."); + } + + public boolean isEdgeUnit() { + throw new IllegalStateException("This method should not be called for DiphoneUnits."); + } + + public String toString() { + return "diphoneunit " + index + " start: " + startTime + ", duration: " + duration; + } + + /** + * inspired by http://www.artima.com/lejava/articles/equality.html + */ + @Override + public boolean equals(Object other) { + boolean result = false; + if (other instanceof DiphoneUnit) { + DiphoneUnit that = (DiphoneUnit) other; + result = (this.left.equals(that.left) && this.right.equals(that.right)); + } + return result; + } + + /** + * inspired by http://www.artima.com/lejava/articles/equality.html + */ + @Override + public int hashCode() { + return (41 * (41 + this.left.hashCode()) + this.right.hashCode()); + } + +} diff --git a/marytts-unitselection/src/main/java/marytts/unitselection/data/DiphoneUnitDatabase.java b/marytts-unitselection/src/main/java/marytts/unitselection/data/DiphoneUnitDatabase.java new file mode 100644 index 0000000000..155a52d9e9 --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/unitselection/data/DiphoneUnitDatabase.java @@ -0,0 +1,159 @@ +/** + * Copyright 2006 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.unitselection.data; + +import gnu.trove.TIntHashSet; + +import java.util.ArrayList; +import java.util.List; + +import marytts.features.FeatureVector; +import marytts.features.DiphoneTarget; +import marytts.features.HalfPhoneTarget; +import marytts.features.Target; +import marytts.unitselection.select.viterbi.ViterbiCandidate; +import marytts.util.MaryUtils; +import marytts.util.dom.DomUtils; + +import org.w3c.dom.Element; + +public class DiphoneUnitDatabase extends UnitDatabase { + + public DiphoneUnitDatabase() { + super(); + logger = MaryUtils.getLogger("DiphoneUnitDatabase"); + } + + /** + * Preselect a set of candidates that could be used to realise the given target. + * + * @param target + * a Target object representing an optimal unit + * @return an unsorted ArrayList of ViterbiCandidates, each containing the (same) target and a + * (different) Unit object + */ + @Override + public List getCandidates(Target target) { + if (!(target instanceof DiphoneTarget)) + return super.getCandidates(target); + // Basic idea: get the candidates for each half phone separately, + // but retain only those that are part of a suitable diphone + DiphoneTarget diphoneTarget = (DiphoneTarget) target; + HalfPhoneTarget left = diphoneTarget.left; + HalfPhoneTarget right = diphoneTarget.right; + + // BEGIN blacklisting + // The point of this is to get the value of the "blacklist" attribute in the first child element of the MaryXML + // and store it in the blacklist String variable. + // This code seems rather inelegant; perhaps there is a better way to access the MaryXML from this method? + String blacklist = ""; + String unitBasename = "This must never be null or the empty string!"; // otherwise candidate selection fails! + Element targetElement = left.getMaryxmlElement(); + if (targetElement == null) { + targetElement = right.getMaryxmlElement(); + } + blacklist = DomUtils.getAttributeFromClosestAncestorOfAnyKind(targetElement, "blacklist"); + // END blacklisting + + // TODO shouldn't leftName and rightName just call appropriate methods of DiphoneTarget? + String leftName = left.getName().substring(0, left.getName().lastIndexOf("_")); + String rightName = right.getName().substring(0, right.getName().lastIndexOf("_")); + int iPhoneme = targetCostFunction.getFeatureDefinition().getFeatureIndex("phone"); + byte bleftName = targetCostFunction.getFeatureDefinition().getFeatureValueAsByte(iPhoneme, leftName); + byte brightName = targetCostFunction.getFeatureDefinition().getFeatureValueAsByte(iPhoneme, rightName); + FeatureVector[] fvs = targetCostFunction.getFeatureVectors(); + + // HashSet candidateUnitSet = new HashSet(); + TIntHashSet candidateUnitSet = new TIntHashSet(); + + // Pre-select candidates for the left half, but retain only + // those that belong to appropriate diphones: + int[] clist = (int[]) preselectionCART.interpret(left, backtrace); + logger.debug("For target " + target + ", selected " + clist.length + " units"); + + // Now, clist is an array of halfphone unit indexes. + for (int i = 0; i < clist.length; i++) { + Unit unit = unitReader.units[clist[i]]; + FeatureVector fv = fvs != null ? fvs[unit.index] : targetCostFunction.getFeatureVector(unit); + byte bunitName = fv.byteValuedDiscreteFeatures[iPhoneme]; + // force correct phone symbol: + if (bunitName != bleftName) + continue; + int iRightNeighbour = clist[i] + 1; + if (iRightNeighbour < numUnits) { + Unit rightNeighbour = unitReader.units[iRightNeighbour]; + FeatureVector rfv = fvs != null ? fvs[iRightNeighbour] : targetCostFunction.getFeatureVector(rightNeighbour); + byte brightUnitName = rfv.byteValuedDiscreteFeatures[iPhoneme]; + if (brightUnitName == brightName) { + // Found a diphone -- add it to candidates + // DiphoneUnit diphoneUnit = new DiphoneUnit(unit, rightNeighbour); + // candidateUnitSet.add(diphoneUnit); + candidateUnitSet.add(unit.index); + } + } + } + // Pre-select candidates for the right half, but retain only + // those that belong to appropriate diphones: + clist = (int[]) preselectionCART.interpret(right, backtrace); + logger.debug("For target " + target + ", selected " + clist.length + " units"); + + // Now, clist is an array of halfphone unit indexes. + for (int i = 0; i < clist.length; i++) { + Unit unit = unitReader.units[clist[i]]; + FeatureVector fv = fvs != null ? fvs[unit.index] : targetCostFunction.getFeatureVector(unit); + byte bunitName = fv.byteValuedDiscreteFeatures[iPhoneme]; + // force correct phone symbol: + if (bunitName != brightName) + continue; + int iLeftNeighbour = clist[i] - 1; + if (iLeftNeighbour >= 0) { + Unit leftNeighbour = unitReader.units[iLeftNeighbour]; + FeatureVector lfv = fvs != null ? fvs[iLeftNeighbour] : targetCostFunction.getFeatureVector(leftNeighbour); + byte bleftUnitName = lfv.byteValuedDiscreteFeatures[iPhoneme]; + if (bleftUnitName == bleftName) { + // Found a diphone -- add it to candidates + // DiphoneUnit diphoneUnit = new DiphoneUnit(leftNeighbour, unit); + // candidateUnitSet.add(diphoneUnit); + candidateUnitSet.add(leftNeighbour.index); + } + } + } + + // now create ArrayList of ViterbiCandidates from the candidateUnitSet, blacklisting along the way: + ArrayList candidates = new ArrayList(candidateUnitSet.size()); + for (int leftIndex : candidateUnitSet.toArray()) { + DiphoneUnit diphoneUnit = new DiphoneUnit(unitReader.units[leftIndex], unitReader.units[leftIndex + 1]); + ViterbiCandidate candidate = new ViterbiCandidate(diphoneTarget, diphoneUnit, targetCostFunction); + // Blacklisting: + if (blacklist.equals("")) { // no blacklist + candidates.add(candidate); + } else { // maybe exclude candidate + unitBasename = getFilename(diphoneUnit); + if (!blacklist.contains(unitBasename)) { + candidates.add(candidate); + } + } + } + + logger.debug("Preselected " + candidateUnitSet.size() + " diphone candidates for target " + target); + return candidates; + } + +} diff --git a/marytts-unitselection/src/main/java/marytts/unitselection/data/FeatureFileReader.java b/marytts-unitselection/src/main/java/marytts/unitselection/data/FeatureFileReader.java new file mode 100644 index 0000000000..9ce3e6e136 --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/unitselection/data/FeatureFileReader.java @@ -0,0 +1,218 @@ +/** + * Copyright 2006 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.unitselection.data; + +import java.io.BufferedInputStream; +import java.io.DataInputStream; +import java.io.FileInputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; + +import marytts.exceptions.MaryConfigurationException; +import marytts.features.FeatureDefinition; +import marytts.features.FeatureVector; +import marytts.util.data.MaryHeader; + +public class FeatureFileReader { + protected MaryHeader hdr; + protected FeatureDefinition featureDefinition; + protected FeatureVector[] featureVectors; + + /** + * Get a feature file reader representing the given feature file. + * + * @param fileName + * the filename of a valid feature file. + * @return a feature file object representing the given file. + * @throws IOException + * if there was a problem reading the file + * @throws MaryConfigurationException + * if the file is not a valid feature file. + */ + public static FeatureFileReader getFeatureFileReader(String fileName) throws IOException, MaryConfigurationException { + int fileType = MaryHeader.peekFileType(fileName); + if (fileType == MaryHeader.UNITFEATS) + return new FeatureFileReader(fileName); + else if (fileType == MaryHeader.HALFPHONE_UNITFEATS) + return new HalfPhoneFeatureFileReader(fileName); + throw new MaryConfigurationException("File " + fileName + ": Type " + fileType + " is not a known unit feature file type"); + } + + /** + * Empty constructor; need to call load() separately when using this. + * + * @see load(String) + */ + public FeatureFileReader() { + } + + public FeatureFileReader(String fileName) throws IOException, MaryConfigurationException { + load(fileName); + } + + public void load(String fileName) throws IOException, MaryConfigurationException { + loadFromByteBuffer(fileName); + } + + protected void loadFromStream(String fileName) throws IOException, MaryConfigurationException { + /* Open the file */ + DataInputStream dis = null; + dis = new DataInputStream(new BufferedInputStream(new FileInputStream(fileName))); + + /* Load the Mary header */ + hdr = new MaryHeader(dis); + if (hdr.getType() != MaryHeader.UNITFEATS && hdr.getType() != MaryHeader.HALFPHONE_UNITFEATS) { + throw new IOException("File [" + fileName + "] is not a valid Mary feature file."); + } + featureDefinition = new FeatureDefinition(dis); + int numberOfUnits = dis.readInt(); + featureVectors = new FeatureVector[numberOfUnits]; + for (int i = 0; i < numberOfUnits; i++) { + featureVectors[i] = featureDefinition.readFeatureVector(i, dis); + } + + } + + protected void loadFromByteBuffer(String fileName) throws IOException, MaryConfigurationException { + /* Open the file */ + FileInputStream fis = new FileInputStream(fileName); + FileChannel fc = fis.getChannel(); + ByteBuffer bb = fc.map(FileChannel.MapMode.READ_ONLY, 0, fc.size()); + fis.close(); + + /* Load the Mary header */ + hdr = new MaryHeader(bb); + if (hdr.getType() != MaryHeader.UNITFEATS && hdr.getType() != MaryHeader.HALFPHONE_UNITFEATS) { + throw new MaryConfigurationException("File [" + fileName + "] is not a valid Mary feature file."); + } + featureDefinition = new FeatureDefinition(bb); + int numberOfUnits = bb.getInt(); + featureVectors = new FeatureVector[numberOfUnits]; + for (int i = 0; i < numberOfUnits; i++) { + featureVectors[i] = featureDefinition.readFeatureVector(i, bb); + } + + } + + /** + * Get the unit feature vector for the given unit index number. + * + * @param unitIndex + * the absolute index number of a unit in the database + * @return the corresponding feature vector + */ + public FeatureVector getFeatureVector(int unitIndex) { + return featureVectors[unitIndex]; + } + + /** + * Return a shallow copy of the array of feature vectors. + * + * @return a new array containing the internal feature vectors + */ + public FeatureVector[] getCopyOfFeatureVectors() { + return (FeatureVector[]) featureVectors.clone(); + } + + /** + * Return the internal array of feature vectors. + * + * @return the internal array of feature vectors. + */ + public FeatureVector[] getFeatureVectors() { + return featureVectors; + } + + /** + * feature vector mapping according to new feature definition Note: The new feature definition should be a subset of original + * feature definition + * + * @param newFeatureDefinition + * @return + */ + public FeatureVector[] featureVectorMapping(FeatureDefinition newFeatureDefinition) { + + if (!this.featureDefinition.contains(newFeatureDefinition)) { + throw new RuntimeException("the new feature definition is not a subset of original feature definition"); + } + + int numberOfFeatures = newFeatureDefinition.getNumberOfFeatures(); + int noByteFeatures = newFeatureDefinition.getNumberOfByteFeatures(); + int noShortFeatures = newFeatureDefinition.getNumberOfShortFeatures(); + int noContiniousFeatures = newFeatureDefinition.getNumberOfContinuousFeatures(); + + if (numberOfFeatures != (noByteFeatures + noShortFeatures + noContiniousFeatures)) { + throw new RuntimeException("The sum of byte, short and continious features are not equal to number of features"); + } + + String[] featureNames = new String[numberOfFeatures]; + for (int j = 0; j < numberOfFeatures; j++) { + featureNames[j] = newFeatureDefinition.getFeatureName(j); + } + int[] featureIndexes = featureDefinition.getFeatureIndexArray(featureNames); + FeatureVector[] newFV = new FeatureVector[this.getNumberOfUnits()]; + + for (int i = 0; i < this.getNumberOfUnits(); i++) { + + // create features array + byte[] byteFeatures = new byte[noByteFeatures]; + short[] shortFeatures = new short[noShortFeatures]; + float[] continiousFeatures = new float[noContiniousFeatures]; + + int countByteFeatures = 0; + int countShortFeatures = 0; + int countFloatFeatures = 0; + + for (int j = 0; j < featureIndexes.length; j++) { + if (newFeatureDefinition.isByteFeature(j)) { + byteFeatures[countByteFeatures++] = featureVectors[i].getByteFeature(featureIndexes[j]); + } else if (newFeatureDefinition.isShortFeature(j)) { + shortFeatures[countShortFeatures++] = featureVectors[i].getShortFeature(featureIndexes[j]); + } else if (newFeatureDefinition.isContinuousFeature(j)) { + continiousFeatures[countFloatFeatures++] = featureVectors[i].getContinuousFeature(featureIndexes[j]); + } + } + + newFV[i] = newFeatureDefinition.toFeatureVector(i, byteFeatures, shortFeatures, continiousFeatures); + } + + return newFV; + } + + /** + * Get the unit feature vector for the given unit. + * + * @param unit + * a unit in the database + * @return the corresponding feature vector + */ + public FeatureVector getFeatureVector(Unit unit) { + return featureVectors[unit.index]; + } + + public FeatureDefinition getFeatureDefinition() { + return featureDefinition; + } + + public int getNumberOfUnits() { + return (featureVectors.length); + } +} diff --git a/marytts-unitselection/src/main/java/marytts/unitselection/data/FloatArrayDatagram.java b/marytts-unitselection/src/main/java/marytts/unitselection/data/FloatArrayDatagram.java new file mode 100644 index 0000000000..1036e6a049 --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/unitselection/data/FloatArrayDatagram.java @@ -0,0 +1,67 @@ +/** + * Copyright 2010 DFKI GmbH. All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License along with this program. If not, see + * . + * + */ + +package marytts.unitselection.data; + +import java.io.BufferedOutputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataOutput; +import java.io.DataOutputStream; +import java.io.IOException; + +import marytts.util.data.Datagram; + +/** + * Extension of Datagram to provide a float array instead of (actually alongside) a byte array + * + * @author steiner + * + */ +public class FloatArrayDatagram extends Datagram { + + private float[] floatData; + + public FloatArrayDatagram(long duration, float[] data) { + super(duration); + this.floatData = data; + } + + public float[] getFloatData() { + return floatData; + } + + /** + * Write this datagram to a random access file or data output stream. + * + * @throws IOException + */ + @Override + public void write(DataOutput raf) throws IOException { + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream dos = new DataOutputStream(baos); + BufferedOutputStream bos = new BufferedOutputStream(dos); + + dos.writeLong(duration); + dos.writeInt(floatData.length); + for (float fl : floatData) { + dos.writeFloat(fl); + } + + raf.write(baos.toByteArray()); + } + +} diff --git a/marytts-unitselection/src/main/java/marytts/unitselection/data/HalfPhoneFeatureFileReader.java b/marytts-unitselection/src/main/java/marytts/unitselection/data/HalfPhoneFeatureFileReader.java new file mode 100644 index 0000000000..fd3775e6f6 --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/unitselection/data/HalfPhoneFeatureFileReader.java @@ -0,0 +1,99 @@ +/** + * Copyright 2006 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.unitselection.data; + +import java.io.BufferedInputStream; +import java.io.DataInputStream; +import java.io.FileInputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; + +import marytts.exceptions.MaryConfigurationException; +import marytts.features.FeatureDefinition; +import marytts.features.FeatureVector; +import marytts.util.data.MaryHeader; + +public class HalfPhoneFeatureFileReader extends FeatureFileReader { + protected FeatureDefinition leftWeights; + protected FeatureDefinition rightWeights; + + public HalfPhoneFeatureFileReader() { + super(); + } + + public HalfPhoneFeatureFileReader(String fileName) throws IOException, MaryConfigurationException { + super(fileName); + } + + @Override + protected void loadFromStream(String fileName) throws IOException, MaryConfigurationException { + /* Open the file */ + DataInputStream dis = null; + dis = new DataInputStream(new BufferedInputStream(new FileInputStream(fileName))); + /* Load the Mary header */ + hdr = new MaryHeader(dis); + if (hdr.getType() != MaryHeader.HALFPHONE_UNITFEATS) { + throw new IOException("File [" + fileName + "] is not a valid Mary Halfphone Features file."); + } + leftWeights = new FeatureDefinition(dis); + rightWeights = new FeatureDefinition(dis); + assert leftWeights.featureEquals(rightWeights) : "Halfphone unit feature file contains incompatible feature definitions for left and right units -- this should not happen!"; + featureDefinition = leftWeights; // one of them, for super class + int numberOfUnits = dis.readInt(); + featureVectors = new FeatureVector[numberOfUnits]; + for (int i = 0; i < numberOfUnits; i++) { + featureVectors[i] = featureDefinition.readFeatureVector(i, dis); + } + } + + @Override + protected void loadFromByteBuffer(String fileName) throws IOException, MaryConfigurationException { + /* Open the file */ + FileInputStream fis = new FileInputStream(fileName); + FileChannel fc = fis.getChannel(); + ByteBuffer bb = fc.map(FileChannel.MapMode.READ_ONLY, 0, fc.size()); + fis.close(); + + /* Load the Mary header */ + hdr = new MaryHeader(bb); + if (hdr.getType() != MaryHeader.HALFPHONE_UNITFEATS) { + throw new MaryConfigurationException("File [" + fileName + "] is not a valid Mary Halfphone Features file."); + } + leftWeights = new FeatureDefinition(bb); + rightWeights = new FeatureDefinition(bb); + assert leftWeights.featureEquals(rightWeights) : "Halfphone unit feature file contains incompatible feature definitions for left and right units -- this should not happen!"; + featureDefinition = leftWeights; // one of them, for super class + int numberOfUnits = bb.getInt(); + featureVectors = new FeatureVector[numberOfUnits]; + for (int i = 0; i < numberOfUnits; i++) { + featureVectors[i] = featureDefinition.readFeatureVector(i, bb); + } + } + + public FeatureDefinition getLeftWeights() { + return leftWeights; + } + + public FeatureDefinition getRightWeights() { + return rightWeights; + } + +} diff --git a/marytts-unitselection/src/main/java/marytts/unitselection/data/HnmDatagram.java b/marytts-unitselection/src/main/java/marytts/unitselection/data/HnmDatagram.java new file mode 100644 index 0000000000..35ca5bd246 --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/unitselection/data/HnmDatagram.java @@ -0,0 +1,149 @@ +/** + * Copyright 2006 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.unitselection.data; + +import java.io.ByteArrayInputStream; +import java.io.DataInputStream; +import java.io.DataOutput; +import java.io.EOFException; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; + +import marytts.signalproc.sinusoidal.hntm.analysis.HntmSpeechFrame; +import marytts.util.data.Datagram; + +/** + * A datagram that encapsulates a harmonics plus noise modelled speech frame + * + * @author Oytun Türk + * + */ +public class HnmDatagram extends Datagram { + + public HntmSpeechFrame frame; // Hnm parameters for a speech frame + + /** + * Construct a HNM datagram. + * + * @param duration + * the duration, in samples, of the data represented by this datagram + * @param frame + * the parameters of HNM for a speech frame. + */ + public HnmDatagram(long setDuration, HntmSpeechFrame frame) { + super(setDuration); + + this.frame = new HntmSpeechFrame(frame); + } + + /** + * Constructor which pops a datagram from a random access file. + * + * @param raf + * the random access file to pop the datagram from. + * + * @throws IOException + * @throws EOFException + */ + public HnmDatagram(RandomAccessFile raf, int noiseModel) throws IOException, EOFException { + super(raf.readLong()); // duration + int len = raf.readInt(); + if (len < 0) { + throw new IOException("Can't create a datagram with a negative data size [" + len + "]."); + } + if (len < 4 * 3) { + throw new IOException("Hnm with waveform noise datagram too short (len=" + len + + "): cannot be shorter than the space needed for first three Hnm parameters (4*3)"); + } + + // For speed concerns, read into a byte[] first: + byte[] buf = new byte[len]; + raf.readFully(buf); + DataInputStream dis = new DataInputStream(new ByteArrayInputStream(buf)); + + frame = new HntmSpeechFrame(dis, noiseModel); + } + + /** + * Constructor which pops a datagram from a byte buffer. + * + * @param bb + * the byte buffer to pop the datagram from. + * + * @throws IOException + * @throws EOFException + */ + public HnmDatagram(ByteBuffer bb, int noiseModel) throws IOException, EOFException { + super(bb.getLong()); // duration + int len = bb.getInt(); + if (len < 0) { + throw new IOException("Can't create a datagram with a negative data size [" + len + "]."); + } + if (len < 4 * 3) { + throw new IOException("Hnm with waveform noise datagram too short (len=" + len + + "): cannot be shorter than the space needed for first three Hnm parameters (4*3)"); + } + + frame = new HntmSpeechFrame(bb, noiseModel); + } + + /** + * Get the length, in bytes, of the datagram's data field. + */ + public int getLength() { + return frame.getLength(); + } + + /** + * Get the sinusoidal speech frame + * + * @return frame + */ + public HntmSpeechFrame getFrame() { + return frame; + } + + /** + * Write this datagram to a random access file or data output stream. + */ + public void write(DataOutput out) throws IOException { + out.writeLong(duration); + out.writeInt(getLength()); + + frame.write(out); + } + + /** + * Tests if this datagram is equal to another datagram. + */ + public boolean equals(Datagram other) { + if (!(other instanceof HnmDatagram)) + return false; + HnmDatagram otherHnm = (HnmDatagram) other; + if (this.duration != otherHnm.duration) + return false; + if (!this.frame.equals(otherHnm.frame)) + return false; + + return true; + } + +} diff --git a/marytts-unitselection/src/main/java/marytts/unitselection/data/HnmTimelineReader.java b/marytts-unitselection/src/main/java/marytts/unitselection/data/HnmTimelineReader.java new file mode 100644 index 0000000000..b9c307b1be --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/unitselection/data/HnmTimelineReader.java @@ -0,0 +1,248 @@ +/** + * Copyright 2006 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.unitselection.data; + +import java.io.ByteArrayInputStream; +import java.io.DataOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.LinkedList; +import java.util.Properties; + +import javax.sound.sampled.UnsupportedAudioFileException; + +import marytts.exceptions.MaryConfigurationException; +import marytts.signalproc.adaptation.prosody.BasicProsodyModifierParams; +import marytts.signalproc.sinusoidal.hntm.analysis.FrameNoisePartWaveform; +import marytts.signalproc.sinusoidal.hntm.analysis.HntmAnalyzerParams; +import marytts.signalproc.sinusoidal.hntm.analysis.HntmSpeechFrame; +import marytts.signalproc.sinusoidal.hntm.analysis.HntmSpeechSignal; +import marytts.signalproc.sinusoidal.hntm.synthesis.HntmSynthesizedSignal; +import marytts.signalproc.sinusoidal.hntm.synthesis.HntmSynthesizer; +import marytts.signalproc.sinusoidal.hntm.synthesis.HntmSynthesizerParams; +import marytts.util.data.Datagram; +import marytts.util.io.FileUtils; +import marytts.util.math.ArrayUtils; +import marytts.util.math.MathUtils; + +/** + * A reader class for the harmonics plus noise timeline file. + * + * @author Oytun Türk + * + */ +public class HnmTimelineReader extends TimelineReader { + public HntmAnalyzerParams analysisParams; + + public HnmTimelineReader(String fileName) throws IOException, MaryConfigurationException { + super(); + load(fileName); + } + + protected void load(String fileName) throws IOException, MaryConfigurationException { + super.load(fileName); + // Now make sense of the processing header + Properties props = new Properties(); + ByteArrayInputStream bais = new ByteArrayInputStream(procHdr.getString().getBytes("latin1")); + props.load(bais); + ensurePresent(props, "hnm.noiseModel"); + + analysisParams = new HntmAnalyzerParams(); + + analysisParams.noiseModel = Integer.parseInt(props.getProperty("hnm.noiseModel")); + analysisParams.hnmPitchVoicingAnalyzerParams.numFilteringStages = Integer + .parseInt(props.getProperty("hnm.numFiltStages")); + analysisParams.hnmPitchVoicingAnalyzerParams.medianFilterLength = Integer + .parseInt(props.getProperty("hnm.medianFiltLen")); + analysisParams.hnmPitchVoicingAnalyzerParams.movingAverageFilterLength = Integer.parseInt(props + .getProperty("hnm.maFiltLen")); + analysisParams.hnmPitchVoicingAnalyzerParams.cumulativeAmpThreshold = Float.parseFloat(props.getProperty("hnm.cumAmpTh")); + analysisParams.hnmPitchVoicingAnalyzerParams.maximumAmpThresholdInDB = Float + .parseFloat(props.getProperty("hnm.maxAmpTh")); + analysisParams.hnmPitchVoicingAnalyzerParams.harmonicDeviationPercent = Float.parseFloat(props + .getProperty("hnm.harmDevPercent")); + analysisParams.hnmPitchVoicingAnalyzerParams.sharpPeakAmpDiffInDB = Float.parseFloat(props + .getProperty("hnm.sharpPeakAmpDiff")); + analysisParams.hnmPitchVoicingAnalyzerParams.minimumTotalHarmonics = Integer.parseInt(props + .getProperty("hnm.minHarmonics")); + analysisParams.hnmPitchVoicingAnalyzerParams.maximumTotalHarmonics = Integer.parseInt(props + .getProperty("hnm.maxHarmonics")); + analysisParams.hnmPitchVoicingAnalyzerParams.minimumVoicedFrequencyOfVoicing = Float.parseFloat(props + .getProperty("hnm.minVoicedFreq")); + analysisParams.hnmPitchVoicingAnalyzerParams.maximumVoicedFrequencyOfVoicing = Float.parseFloat(props + .getProperty("hnm.maxVoicedFreq")); + analysisParams.hnmPitchVoicingAnalyzerParams.maximumFrequencyOfVoicingFinalShift = Float.parseFloat(props + .getProperty("hnm.maxFreqVoicingFinalShift")); + analysisParams.hnmPitchVoicingAnalyzerParams.neighsPercent = Float.parseFloat(props.getProperty("hnm.neighsPercent")); + analysisParams.harmonicPartCepstrumOrder = Integer.parseInt(props.getProperty("hnm.harmCepsOrder")); + analysisParams.regularizedCepstrumWarpingMethod = Integer.parseInt(props.getProperty("hnm.regCepWarpMethod")); + analysisParams.regularizedCepstrumLambdaHarmonic = Float.parseFloat(props.getProperty("hnm.regCepsLambda")); + analysisParams.noisePartLpOrder = Integer.parseInt(props.getProperty("hnm.noiseLpOrder")); + analysisParams.preemphasisCoefNoise = Float.parseFloat(props.getProperty("hnm.preCoefNoise")); + analysisParams.hpfBeforeNoiseAnalysis = Boolean.parseBoolean(props.getProperty("hnm.hpfBeforeNoiseAnalysis")); + analysisParams.numPeriodsHarmonicsExtraction = Float.parseFloat(props.getProperty("hnm.harmNumPer")); + } + + private void ensurePresent(Properties props, String key) throws MaryConfigurationException { + if (!props.containsKey(key)) + throw new MaryConfigurationException("Processing header does not contain required field '" + key + "'"); + } + + /** + * {@inheritDoc} + */ + @Override + protected Datagram getNextDatagram(ByteBuffer bb) { + + Datagram d = null; + + /* If the end of the datagram zone is reached, refuse to read */ + if (bb.position() == bb.limit()) { + // throw new IndexOutOfBoundsException( "Time out of bounds: you are trying to read a datagram at" + + // " a time which is bigger than the total timeline duration." ); + return null; + } + /* Else, pop the datagram out of the file */ + try { + d = new HnmDatagram(bb, analysisParams.noiseModel); + } + /* Detect a possible EOF encounter */ + catch (IOException e) { + return null; + } + + return d; + } + + private void testSynthesizeFromDatagrams(LinkedList datagrams, int startIndex, int endIndex, + DataOutputStream output) throws IOException { + HntmSynthesizer s = new HntmSynthesizer(); + // TODO: These should come from timeline and user choices... + HntmAnalyzerParams hnmAnalysisParams = new HntmAnalyzerParams(); + HntmSynthesizerParams synthesisParams = new HntmSynthesizerParams(); + BasicProsodyModifierParams pmodParams = new BasicProsodyModifierParams(); + int samplingRateInHz = this.getSampleRate(); + + int totalFrm = 0; + int i; + float originalDurationInSeconds = 0.0f; + float deltaTimeInSeconds; + + for (i = startIndex; i <= endIndex; i++) { + HnmDatagram datagram; + try { + datagram = datagrams.get(i); + } catch (IndexOutOfBoundsException e) { + throw e; + } + if (datagram != null && datagram instanceof HnmDatagram) { + totalFrm++; + // deltaTimeInSeconds = SignalProcUtils.sample2time(((HnmDatagram)datagrams.get(i)).getDuration(), + // samplingRateInHz); + deltaTimeInSeconds = datagram.frame.deltaAnalysisTimeInSeconds; + originalDurationInSeconds += deltaTimeInSeconds; + } + } + + HntmSpeechSignal hnmSignal = new HntmSpeechSignal(totalFrm, samplingRateInHz, originalDurationInSeconds); + + int frameCount = 0; + float tAnalysisInSeconds = 0.0f; + for (i = startIndex; i <= endIndex; i++) { + HnmDatagram datagram; + try { + datagram = datagrams.get(i); + } catch (IndexOutOfBoundsException e) { + throw e; + } + if (datagram != null && datagram instanceof HnmDatagram) { + // tAnalysisInSeconds += SignalProcUtils.sample2time(((HnmDatagram)datagrams.get(i)).getDuration(), + // samplingRateInHz); + tAnalysisInSeconds += datagram.getFrame().deltaAnalysisTimeInSeconds; + + if (frameCount < totalFrm) { + hnmSignal.frames[frameCount] = new HntmSpeechFrame(datagram.getFrame()); + hnmSignal.frames[frameCount].tAnalysisInSeconds = tAnalysisInSeconds; + frameCount++; + } + } + } + + HntmSynthesizedSignal ss = null; + if (totalFrm > 0) { + ss = s.synthesize(hnmSignal, null, null, pmodParams, null, hnmAnalysisParams, synthesisParams); + FileUtils.writeBinaryFile(ArrayUtils.copyDouble2Short(ss.output), output); + if (ss.output != null) { + ss.output = MathUtils.multiply(ss.output, 1.0 / 32768.0); // why is this done here? + } + } + return; + } + + /** + * Dump audio from HNM timeline to a series big-endian raw audio files in chunks of Datagrams (clusterSize). Run this + * with + * + * 
+	 * -ea - Xmx2gb
+	 *
+ * + * @param args + *
    + *
  1. path to timeline_hnm.mry file
    2. + *
  2. path to dump output files
    3. + *
+ * @throws IOException + */ + public static void main(String[] args) throws UnsupportedAudioFileException, IOException, MaryConfigurationException { + HnmTimelineReader h = new HnmTimelineReader(args[0]); + + LinkedList datagrams = new LinkedList(); + int count = 0; + long startDatagramTime = 0; + int numDatagrams = (int) h.numDatagrams; + // long numDatagrams = 2000; + + Datagram[] rawDatagrams = h.getDatagrams(0l, numDatagrams, h.getSampleRate(), null); + for (int i = 0; i < rawDatagrams.length; i++) { + HnmDatagram d = (HnmDatagram) rawDatagrams[i]; + datagrams.add(d); + count++; + System.out.println("Datagram " + String.valueOf(count) + "Noise waveform size=" + + ((FrameNoisePartWaveform) (((HnmDatagram) d).frame.n)).waveform().length); + } + + int clusterSize = 1000; + int numClusters = (int) Math.floor((numDatagrams) / ((double) clusterSize) + 0.5); + int startIndex, endIndex; + for (int i = 0; i < numClusters; i++) { + DataOutputStream output = new DataOutputStream(new FileOutputStream( + new File(String.format("%s_%06d.bin", args[1], i)))); + startIndex = (int) (i * clusterSize); + endIndex = (int) Math.min((i + 1) * clusterSize - 1, numDatagrams - 1); + h.testSynthesizeFromDatagrams(datagrams, startIndex, endIndex, output); + System.out.println("Timeline cluster " + String.valueOf(i + 1) + " of " + String.valueOf(numClusters) + + " synthesized..."); + output.close(); + } + } +} diff --git a/marytts-unitselection/src/main/java/marytts/unitselection/data/LPCDatagram.java b/marytts-unitselection/src/main/java/marytts/unitselection/data/LPCDatagram.java new file mode 100644 index 0000000000..ed52c357b1 --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/unitselection/data/LPCDatagram.java @@ -0,0 +1,229 @@ +/** + * Copyright 2006 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.unitselection.data; + +import java.io.ByteArrayInputStream; +import java.io.DataInputStream; +import java.io.DataOutput; +import java.io.EOFException; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; + +import marytts.util.data.Datagram; +import marytts.util.io.General; + +public class LPCDatagram extends Datagram { + protected short[] quantizedCoeffs; + protected byte[] quantizedResidual; + + /** + * Construct an LPC datagram from quantized data. + * + * @param duration + * the duration, in samples, of the data represented by this datagram + * @param quantizedCoeffs + * the quantized LPC coefficients + * @param quantizedResidual + * the quantized residual + */ + public LPCDatagram(long setDuration, short[] quantizedCoeffs, byte[] quantizedResidual) { + super(setDuration); + this.quantizedCoeffs = quantizedCoeffs; + this.quantizedResidual = quantizedResidual; + } + + /** + * Construct an LPC datagram from unquantized data. + * + * @param duration + * the duration, in samples, of the data represented by this datagram + * @param coeffs + * the (unquantized) LPC coefficients + * @param residual + * the (unquantized) residual + */ + public LPCDatagram(long setDuration, float[] coeffs, short[] residual, float lpcMin, float lpcRange) { + super(setDuration); + this.quantizedCoeffs = General.quantize(coeffs, lpcMin, lpcRange); + this.quantizedResidual = General.shortToUlaw(residual); + } + + /** + * Constructor which pops a datagram from a random access file. + * + * @param raf + * the random access file to pop the datagram from. + * + * @throws IOException + * @throws EOFException + */ + public LPCDatagram(RandomAccessFile raf, int lpcOrder) throws IOException, EOFException { + super(raf.readLong()); // duration + int len = raf.readInt(); + if (len < 0) { + throw new IOException("Can't create a datagram with a negative data size [" + len + "]."); + } + if (len < 2 * lpcOrder) { + throw new IOException("LPC datagram too short (len=" + len + + "): cannot be shorter than the space needed for lpc coefficients (2*" + lpcOrder + ")"); + } + // For speed concerns, read into a byte[] first: + byte[] buf = new byte[len]; + raf.readFully(buf); + DataInputStream dis = new DataInputStream(new ByteArrayInputStream(buf)); + + int residualLength = len - 2 * lpcOrder; + quantizedCoeffs = new short[lpcOrder]; + quantizedResidual = new byte[residualLength]; + for (int i = 0; i < lpcOrder; i++) { + quantizedCoeffs[i] = dis.readShort(); + } + System.arraycopy(buf, 2 * lpcOrder, quantizedResidual, 0, residualLength); + } + + /** + * Constructor which pops a datagram from a byte buffer. + * + * @param bb + * the byte buffer to pop the datagram from. + * + * @throws IOException + * @throws EOFException + */ + public LPCDatagram(ByteBuffer bb, int lpcOrder) throws IOException, EOFException { + super(bb.getLong()); // duration + int len = bb.getInt(); + if (len < 0) { + throw new IOException("Can't create a datagram with a negative data size [" + len + "]."); + } + if (len < 2 * lpcOrder) { + throw new IOException("LPC datagram too short (len=" + len + + "): cannot be shorter than the space needed for lpc coefficients (2*" + lpcOrder + ")"); + } + + int residualLength = len - 2 * lpcOrder; + quantizedCoeffs = new short[lpcOrder]; + quantizedResidual = new byte[residualLength]; + for (int i = 0; i < lpcOrder; i++) { + quantizedCoeffs[i] = bb.getShort(); + } + bb.get(quantizedResidual); + } + + /** + * Get the length, in bytes, of the datagram's data field. + */ + public int getLength() { + return 2 * quantizedCoeffs.length + quantizedResidual.length; + } + + /** + * Get the LPC order, i.e. the number of LPC coefficients. + * + * @return the lpc order + * @see #getQuantizedCoeffs() + * @see #getCoeffs() + */ + public int lpcOrder() { + return quantizedCoeffs.length; + } + + /** + * Get the quantized lpc coefficients + * + * @return an array of shorts, length lpcOrder() + * @see #lpcOrder() + * @see #getCoeffs() + */ + public short[] getQuantizedCoeffs() { + return quantizedCoeffs; + } + + /** + * Get the quantized residual. + * + * @return an array of bytes + */ + public byte[] getQuantizedResidual() { + return quantizedResidual; + } + + /** + * Get the LPC coefficients, unquantized using the given lpc min and range values. + * + * @param lpcMin + * the lpc minimum + * @param lpcRange + * the lpc range + * @return an array of floats, length lpcOrder() + * @see #lpcOrder() + * @see #getQuantizedCoeffs() + */ + public float[] getCoeffs(float lpcMin, float lpcRange) { + return General.unQuantize(quantizedCoeffs, lpcMin, lpcRange); + } + + /** + * Get the unquantized residual + * + * @return an array of shorts + */ + public short[] getResidual() { + return General.ulawToShort(quantizedResidual); + } + + /** + * Write this datagram to a random access file or data output stream. + */ + public void write(DataOutput out) throws IOException { + out.writeLong(duration); + out.writeInt(getLength()); + for (int i = 0; i < quantizedCoeffs.length; i++) { + out.writeShort(quantizedCoeffs[i]); + } + out.write(quantizedResidual); + } + + /** + * Tests if this datagram is equal to another datagram. + */ + public boolean equals(Datagram other) { + if (!(other instanceof LPCDatagram)) + return false; + LPCDatagram otherLPC = (LPCDatagram) other; + if (this.duration != otherLPC.duration) + return false; + if (this.quantizedCoeffs.length != otherLPC.quantizedCoeffs.length) + return false; + if (this.quantizedResidual.length != otherLPC.quantizedResidual.length) + return false; + for (int i = 0; i < this.quantizedCoeffs.length; i++) { + if (this.quantizedCoeffs[i] != otherLPC.quantizedCoeffs[i]) + return false; + } + for (int i = 0; i < this.quantizedResidual.length; i++) { + if (this.quantizedResidual[i] != otherLPC.quantizedResidual[i]) + return false; + } + return true; + } + +} diff --git a/marytts-unitselection/src/main/java/marytts/unitselection/data/LPCTimelineReader.java b/marytts-unitselection/src/main/java/marytts/unitselection/data/LPCTimelineReader.java new file mode 100644 index 0000000000..3fcb84e961 --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/unitselection/data/LPCTimelineReader.java @@ -0,0 +1,100 @@ +/** + * Copyright 2006 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.unitselection.data; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Properties; + +import marytts.exceptions.MaryConfigurationException; +import marytts.util.data.Datagram; + +public class LPCTimelineReader extends TimelineReader { + protected int lpcOrder; + protected float lpcMin; + protected float lpcRange; + + public LPCTimelineReader(String fileName) throws IOException, MaryConfigurationException { + super(); + load(fileName); + } + + @Override + protected void load(String fileName) throws IOException, MaryConfigurationException { + super.load(fileName); + // Now make sense of the processing header + Properties props = new Properties(); + ByteArrayInputStream bais = new ByteArrayInputStream(procHdr.getString().getBytes("latin1")); + props.load(bais); + ensurePresent(props, "lpc.order"); + lpcOrder = Integer.parseInt(props.getProperty("lpc.order")); + ensurePresent(props, "lpc.min"); + lpcMin = Float.parseFloat(props.getProperty("lpc.min")); + ensurePresent(props, "lpc.range"); + lpcRange = Float.parseFloat(props.getProperty("lpc.range")); + } + + private void ensurePresent(Properties props, String key) throws IOException { + if (!props.containsKey(key)) + throw new IOException("Processing header does not contain required field '" + key + "'"); + + } + + public int getLPCOrder() { + return lpcOrder; + } + + public float getLPCMin() { + return lpcMin; + } + + public float getLPCRange() { + return lpcRange; + } + + /** + * Read and return the upcoming datagram. + * + * @return the current datagram, or null if EOF was encountered; internally updates the time pointer. + * + * @throws IOException + */ + @Override + protected Datagram getNextDatagram(ByteBuffer bb) { + + Datagram d = null; + + /* If the end of the datagram zone is reached, gracefully refuse to read */ + if (bb.position() == timeIdxBytePos) + return (null); + /* Else, pop the datagram out of the file */ + try { + d = new LPCDatagram(bb, lpcOrder); + } + /* Detect a possible EOF encounter */ + catch (IOException e) { + return null; + } + + return (d); + } + +} diff --git a/marytts-unitselection/src/main/java/marytts/unitselection/data/MCepDatagram.java b/marytts-unitselection/src/main/java/marytts/unitselection/data/MCepDatagram.java new file mode 100644 index 0000000000..167bc917ca --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/unitselection/data/MCepDatagram.java @@ -0,0 +1,175 @@ +/** + * Copyright 2006 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.unitselection.data; + +import java.io.ByteArrayInputStream; +import java.io.DataInputStream; +import java.io.DataOutput; +import java.io.EOFException; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; + +import marytts.util.data.Datagram; + +public class MCepDatagram extends Datagram { + + protected float[] coeffs; + + /** + * Construct a MCep datagram from a float vector. + * + * @param duration + * the duration, in samples, of the data represented by this datagram + * @param coeffs + * the array of Mel-Cepstrum coefficients. + */ + public MCepDatagram(long setDuration, float[] coeffs) { + super(setDuration); + this.coeffs = coeffs; + } + + /** + * Constructor which pops a datagram from a random access file. + * + * @param raf + * the random access file to pop the datagram from. + * + * @throws IOException + * @throws EOFException + */ + public MCepDatagram(RandomAccessFile raf, int order) throws IOException, EOFException { + super(raf.readLong()); // duration + int len = raf.readInt(); + if (len < 0) { + throw new IOException("Can't create a datagram with a negative data size [" + len + "]."); + } + if (len < 4 * order) { + throw new IOException("Mel-Cepstrum datagram too short (len=" + len + + "): cannot be shorter than the space needed for Mel-Cepstrum coefficients (4*" + order + ")"); + } + // For speed concerns, read into a byte[] first: + byte[] buf = new byte[len]; + raf.readFully(buf); + DataInputStream dis = new DataInputStream(new ByteArrayInputStream(buf)); + + coeffs = new float[order]; + for (int i = 0; i < order; i++) { + coeffs[i] = dis.readFloat(); + } + } + + /** + * Constructor which pops a datagram from a byte buffer. + * + * @param raf + * the byte buffer to pop the datagram from. + * + * @throws IOException + * @throws EOFException + */ + public MCepDatagram(ByteBuffer bb, int order) throws IOException, EOFException { + super(bb.getLong()); // duration + int len = bb.getInt(); + if (len < 0) { + throw new IOException("Can't create a datagram with a negative data size [" + len + "]."); + } + if (len < 4 * order) { + throw new IOException("Mel-Cepstrum datagram too short (len=" + len + + "): cannot be shorter than the space needed for Mel-Cepstrum coefficients (4*" + order + ")"); + } + coeffs = new float[order]; + for (int i = 0; i < order; i++) { + coeffs[i] = bb.getFloat(); + } + } + + /** + * Get the length, in bytes, of the datagram's data field. + */ + public int getLength() { + return 4 * coeffs.length; + } + + /** + * Get the order, i.e. the number of MEl-Cepstrum coefficients. + * + * @return the order + * @see #getCoeffs() + */ + public int order() { + return coeffs.length; + } + + /** + * Get the array of Mel-Cepstrum coefficients. + */ + public float[] getCoeffs() { + return coeffs; + } + + /** + * Get the array of Mel-Cepstrum coefficients. + */ + public double[] getCoeffsAsDouble() { + double[] ret = new double[coeffs.length]; + for (int i = 0; i < coeffs.length; i++) { + ret[i] = (double) (coeffs[i]); + } + return (ret); + } + + /** + * Get a particular Mel-Cepstrum coefficient. + */ + public float getCoeff(int i) { + return coeffs[i]; + } + + /** + * Write this datagram to a random access file or data output stream. + */ + public void write(DataOutput out) throws IOException { + out.writeLong(duration); + out.writeInt(getLength()); + for (int i = 0; i < coeffs.length; i++) { + out.writeFloat(coeffs[i]); + } + } + + /** + * Tests if this datagram is equal to another datagram. + */ + public boolean equals(Datagram other) { + if (!(other instanceof MCepDatagram)) + return false; + MCepDatagram otherMCep = (MCepDatagram) other; + if (this.duration != otherMCep.duration) + return false; + if (this.coeffs.length != otherMCep.coeffs.length) + return false; + for (int i = 0; i < this.coeffs.length; i++) { + if (this.coeffs[i] != otherMCep.coeffs[i]) + return false; + } + return true; + } + +} diff --git a/marytts-unitselection/src/main/java/marytts/unitselection/data/MCepTimelineReader.java b/marytts-unitselection/src/main/java/marytts/unitselection/data/MCepTimelineReader.java new file mode 100644 index 0000000000..cc9e9516c0 --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/unitselection/data/MCepTimelineReader.java @@ -0,0 +1,85 @@ +/** + * Copyright 2006 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.unitselection.data; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Properties; + +import marytts.exceptions.MaryConfigurationException; +import marytts.util.data.Datagram; + +public class MCepTimelineReader extends TimelineReader { + protected int order; + + public MCepTimelineReader(String fileName) throws IOException, MaryConfigurationException { + super(); + load(fileName); + } + + @Override + protected void load(String fileName) throws IOException, MaryConfigurationException { + super.load(fileName); + // Now make sense of the processing header + Properties props = new Properties(); + ByteArrayInputStream bais = new ByteArrayInputStream(procHdr.getString().getBytes("latin1")); + props.load(bais); + ensurePresent(props, "mcep.order"); + order = Integer.parseInt(props.getProperty("mcep.order")); + } + + private void ensurePresent(Properties props, String key) throws IOException { + if (!props.containsKey(key)) + throw new IOException("Processing header does not contain required field '" + key + "'"); + + } + + public int getOrder() { + return order; + } + + /** + * Read and return the upcoming datagram. + * + * @return the current datagram, or null if EOF was encountered; internally updates the time pointer. + * + */ + @Override + protected Datagram getNextDatagram(ByteBuffer bb) { + + Datagram d = null; + + /* If the end of the datagram zone is reached, gracefully refuse to read */ + if (bb.position() == timeIdxBytePos) + return (null); + /* Else, pop the datagram out of the file */ + try { + d = new MCepDatagram(bb, order); + } + /* Detect a possible EOF encounter */ + catch (IOException e) { + return null; + } + + return (d); + } + +} diff --git a/marytts-unitselection/src/main/java/marytts/unitselection/data/SCostFileReader.java b/marytts-unitselection/src/main/java/marytts/unitselection/data/SCostFileReader.java new file mode 100644 index 0000000000..5bd02e274b --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/unitselection/data/SCostFileReader.java @@ -0,0 +1,140 @@ +/** + * Portions Copyright 2006 DFKI GmbH. + * Portions Copyright 2001 Sun Microsystems, Inc. + * Portions Copyright 1999-2001 Language Technologies Institute, + * Carnegie Mellon University. + * All Rights Reserved. Use is subject to license terms. + * + * Permission is hereby granted, free of charge, to use and distribute + * this software and its documentation without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of this work, and to + * permit persons to whom this work is furnished to do so, subject to + * the following conditions: + * + * 1. The code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * 2. Any modifications must be clearly marked as such. + * 3. Original authors' names are not deleted. + * 4. The authors' names are not used to endorse or promote products + * derived from this software without specific prior written + * permission. + * + * DFKI GMBH AND THE CONTRIBUTORS TO THIS WORK DISCLAIM ALL WARRANTIES WITH + * REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL DFKI GMBH NOR THE + * CONTRIBUTORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL + * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR + * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS + * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF + * THIS SOFTWARE. + */ +package marytts.unitselection.data; + +import java.io.BufferedInputStream; +import java.io.DataInputStream; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; + +import marytts.exceptions.MaryConfigurationException; +import marytts.util.data.MaryHeader; + +/** + * sCost file reader + * + * @author sathish pammi + * + */ +public class SCostFileReader { + + private MaryHeader hdr = null; + private int numberOfUnits = 0; + private double[] sCost; + + /****************/ + /* CONSTRUCTORS */ + /****************/ + + /** + * Empty constructor; need to call load() separately. + * + * @see #load(String) + */ + public SCostFileReader() { + } + + /** + * Create a unit file reader from the given unit file + * + * @param fileName + * the unit file to read + * @throws IOException + * if a problem occurs while reading + */ + public SCostFileReader(String fileName) throws IOException, MaryConfigurationException { + load(fileName); + } + + /** + * Load the given unit file + * + * @param fileName + * the unit file to read + * @throws IOException + * if a problem occurs while reading + */ + public void load(String fileName) throws IOException, MaryConfigurationException { + /* Open the file */ + DataInputStream dis = null; + try { + dis = new DataInputStream(new BufferedInputStream(new FileInputStream(fileName))); + } catch (FileNotFoundException e) { + throw new RuntimeException("File [" + fileName + "] was not found."); + } + try { + /* Load the Mary header */ + hdr = new MaryHeader(dis); + if (hdr.getType() != MaryHeader.SCOST) { + throw new RuntimeException("File [" + fileName + "] is not a valid Mary Units file."); + } + /* Read the number of units */ + numberOfUnits = dis.readInt(); + if (numberOfUnits < 0) { + throw new RuntimeException("File [" + fileName + "] has a negative number of units. Aborting."); + } + + sCost = new double[numberOfUnits]; + /* Read the start times and durations */ + for (int i = 0; i < numberOfUnits; i++) { + sCost[i] = dis.readFloat(); + } + } catch (IOException e) { + throw new RuntimeException("Reading the Mary header from file [" + fileName + "] failed.", e); + } + + } + + /*****************/ + /* OTHER METHODS */ + /*****************/ + + /** + * Get the number of units in the file. + * + * @return The number of units. + */ + public int getNumberOfUnits() { + return (numberOfUnits); + } + + /** + * Get sCost for a unit index + * + * @return sCost + */ + public double getSCost(int index) { + return (this.sCost[index]); + } + +} diff --git a/marytts-unitselection/src/main/java/marytts/unitselection/data/Sentence.java b/marytts-unitselection/src/main/java/marytts/unitselection/data/Sentence.java new file mode 100644 index 0000000000..f026ff0168 --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/unitselection/data/Sentence.java @@ -0,0 +1,63 @@ +/** + * Copyright 2010 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ + +package marytts.unitselection.data; + +import java.util.Iterator; + +/** + * This class represents the section of a feature file which constitutes a sentence. + * + * @author marc + * + */ +public class Sentence implements Iterable { + + private FeatureFileReader features; + private int firstUnitIndex; + private int lastUnitIndex; + + public Sentence(FeatureFileReader features, int firstUnitIndex, int lastUnitIndex) { + this.features = features; + this.firstUnitIndex = firstUnitIndex; + this.lastUnitIndex = lastUnitIndex; + } + + public int getFirstUnitIndex() { + return firstUnitIndex; + } + + public int getLastUnitIndex() { + return lastUnitIndex; + } + + public Iterator iterator() { + return new SyllableIterator(features, firstUnitIndex, lastUnitIndex); + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof Sentence)) { + return false; + } + Sentence other = (Sentence) o; + return features.equals(other.features) && firstUnitIndex == other.firstUnitIndex && lastUnitIndex == other.lastUnitIndex; + } +} diff --git a/marytts-unitselection/src/main/java/marytts/unitselection/data/SentenceIterator.java b/marytts-unitselection/src/main/java/marytts/unitselection/data/SentenceIterator.java new file mode 100644 index 0000000000..ed7e60f403 --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/unitselection/data/SentenceIterator.java @@ -0,0 +1,152 @@ +/** + * Copyright 2010 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ + +package marytts.unitselection.data; + +import java.util.Iterator; +import java.util.NoSuchElementException; + +import marytts.features.FeatureDefinition; +import marytts.features.FeatureVector; + +/** + * Iterator to provide the sentences in a given feature file, in sequence. + * + * @author marc + */ +public class SentenceIterator implements Iterator { + + private final FeatureFileReader features; + private final int fiSentenceStart; + private final int fiSentenceEnd; + private final int fiWordStart; + private final int fiWordEnd; + private final boolean isHalfphone; + private final int fiLR; + private final int fvLR_L; + private final int fvLR_R; + + private int i; + private int len; + private Sentence nextSentence = null; + + public SentenceIterator(FeatureFileReader features) { + this.features = features; + FeatureDefinition featureDefinition = features.getFeatureDefinition(); + fiSentenceStart = featureDefinition.getFeatureIndex("words_from_sentence_start"); + fiSentenceEnd = featureDefinition.getFeatureIndex("words_from_sentence_end"); + fiWordStart = featureDefinition.getFeatureIndex("segs_from_word_start"); + fiWordEnd = featureDefinition.getFeatureIndex("segs_from_word_end"); + String halfphoneFeature = "halfphone_lr"; + if (featureDefinition.hasFeature(halfphoneFeature)) { + isHalfphone = true; + fiLR = featureDefinition.getFeatureIndex(halfphoneFeature); + fvLR_L = featureDefinition.getFeatureValueAsByte(fiLR, "L"); + fvLR_R = featureDefinition.getFeatureValueAsByte(fiLR, "R"); + } else { + isHalfphone = false; + fiLR = fvLR_L = fvLR_R = 0; + } + + i = 0; + len = features.getNumberOfUnits(); + } + + public synchronized boolean hasNext() { + if (nextSentence == null) { + prepareNextSentence(); + } + return nextSentence != null; + } + + public synchronized Sentence next() { + if (nextSentence == null) { + prepareNextSentence(); + } + if (nextSentence == null) { + // no more sentences + throw new NoSuchElementException("no more sentences!"); + } + Sentence retval = nextSentence; + nextSentence = null; + return retval; + } + + public void remove() { + throw new UnsupportedOperationException("This iterator cannot remove sentences"); + } + + /** + * Find the next sentence in the feature file, if possible. + */ + private void prepareNextSentence() { + if (nextSentence != null) { + return; + } + if (i >= len) { + return; + } + // if we get here, then i is the index of a unit before or at a sentence start + while (i < len && !isSentenceStart(i)) { + i++; + } + if (i >= len) { + return; + } + int iSentenceStart = i; + while (i < len && !isSentenceEnd(i)) { + i++; + } + if (i >= len) { + return; + } + int iSentenceEnd = i; + nextSentence = new Sentence(features, iSentenceStart, iSentenceEnd); + } + + /** + * Check if the given unit index is a sentence start + * + * @param index + * the unit index + */ + private boolean isSentenceStart(int index) { + FeatureVector fv = features.getFeatureVector(index); + + return fv.getByteFeature(fiSentenceStart) == 0 // first word in sentence + && fv.getByteFeature(fiWordStart) == 0 // first segment in word + && (!isHalfphone || fv.getByteFeature(fiLR) == fvLR_L); // for halfphones, it's the left half + } + + /** + * Check if the given unit index is a sentence end + * + * @param index + * the unit index + */ + private boolean isSentenceEnd(int index) { + FeatureVector fv = features.getFeatureVector(index); + + return fv.getByteFeature(fiSentenceEnd) == 0 // last word in sentence + && fv.getByteFeature(fiWordEnd) == 0 // last segment in word + && (!isHalfphone || fv.getByteFeature(fiLR) == fvLR_R); // for halfphones, it's the right half + } + +} diff --git a/marytts-unitselection/src/main/java/marytts/unitselection/data/Syllable.java b/marytts-unitselection/src/main/java/marytts/unitselection/data/Syllable.java new file mode 100644 index 0000000000..afe14b211e --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/unitselection/data/Syllable.java @@ -0,0 +1,77 @@ +/** + * Copyright 2010 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ + +package marytts.unitselection.data; + +import marytts.features.FeatureVector; + +/** + * This class represents the section of a feature file which constitutes a sentence. + * + * @author marc + * + */ +public class Syllable { + + private FeatureFileReader features; + private int firstUnitIndex; + private int lastUnitIndex; + + public Syllable(FeatureFileReader features, int firstUnitIndex, int lastUnitIndex) { + this.features = features; + this.firstUnitIndex = firstUnitIndex; + this.lastUnitIndex = lastUnitIndex; + } + + public int getFirstUnitIndex() { + return firstUnitIndex; + } + + public int getLastUnitIndex() { + return lastUnitIndex; + } + + /** + * Seek for the syllable nucleus (with feature "ph_vc" == "+") from first to last unit; if none is found, return the last unit + * in the syllable + * + * @return + */ + public int getSyllableNucleusIndex() { + int fiVowel = features.getFeatureDefinition().getFeatureIndex("ph_vc"); + byte fvVowel_Plus = features.getFeatureDefinition().getFeatureValueAsByte(fiVowel, "+"); + for (int i = firstUnitIndex; i <= lastUnitIndex; i++) { + FeatureVector fv = features.getFeatureVector(i); + if (fv.getByteFeature(fiVowel) == fvVowel_Plus) { + return i; + } + } + return lastUnitIndex; + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof Syllable)) { + return false; + } + Syllable other = (Syllable) o; + return features.equals(other.features) && firstUnitIndex == other.firstUnitIndex && lastUnitIndex == other.lastUnitIndex; + } +} diff --git a/marytts-unitselection/src/main/java/marytts/unitselection/data/SyllableIterator.java b/marytts-unitselection/src/main/java/marytts/unitselection/data/SyllableIterator.java new file mode 100644 index 0000000000..1532972449 --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/unitselection/data/SyllableIterator.java @@ -0,0 +1,151 @@ +/** + * Copyright 2009 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ + +package marytts.unitselection.data; + +import java.util.Iterator; +import java.util.NoSuchElementException; + +import marytts.features.FeatureDefinition; +import marytts.features.FeatureVector; + +/** + * @author marc + * + */ +public class SyllableIterator implements Iterator { + + private FeatureFileReader features; + private int fromUnitIndex; + private int toUnitIndex; + + private final int fiPhone; + private final byte fvPhone_0; + private final byte fvPhone_Silence; + private final int fiSylStart; + private final int fiSylEnd; + private final boolean isHalfphone; + private final int fiLR; + private final int fvLR_L; + private final int fvLR_R; + + private int i; + private Syllable nextSyllable = null; + + /** + * Create a syllable iterator over the given feature file, starting from the given fromUnitIndex and reaching up to (and + * including) the given toUnitIndex + * + * @param features + * @param fromUnitIndex + * @param toUnitIndex + */ + public SyllableIterator(FeatureFileReader features, int fromUnitIndex, int toUnitIndex) { + this.features = features; + this.fromUnitIndex = fromUnitIndex; + this.toUnitIndex = toUnitIndex; + + FeatureDefinition featureDefinition = features.getFeatureDefinition(); + fiPhone = featureDefinition.getFeatureIndex("phone"); + fvPhone_0 = featureDefinition.getFeatureValueAsByte(fiPhone, "0"); + fvPhone_Silence = featureDefinition.getFeatureValueAsByte(fiPhone, "_"); + fiSylStart = featureDefinition.getFeatureIndex("segs_from_syl_start"); + fiSylEnd = featureDefinition.getFeatureIndex("segs_from_syl_end"); + String halfphoneFeature = "halfphone_lr"; + if (featureDefinition.hasFeature(halfphoneFeature)) { + isHalfphone = true; + fiLR = featureDefinition.getFeatureIndex(halfphoneFeature); + fvLR_L = featureDefinition.getFeatureValueAsByte(fiLR, "L"); + fvLR_R = featureDefinition.getFeatureValueAsByte(fiLR, "R"); + } else { + isHalfphone = false; + fiLR = fvLR_L = fvLR_R = 0; + } + + i = fromUnitIndex; + } + + public synchronized boolean hasNext() { + if (nextSyllable == null) { + prepareNextSyllable(); + } + return nextSyllable != null; + } + + public synchronized Syllable next() { + if (nextSyllable == null) { + prepareNextSyllable(); + } + if (nextSyllable == null) { + // no more syllables + throw new NoSuchElementException("no more syllables!"); + } + Syllable retval = nextSyllable; + nextSyllable = null; + return retval; + } + + public void remove() { + throw new UnsupportedOperationException("This iterator cannot remove syllables"); + } + + private void prepareNextSyllable() { + if (nextSyllable != null) { + return; + } + if (i > toUnitIndex) { + return; + } + // if we get here, then i is the index of a unit before or at a syllable start + while (i <= toUnitIndex && !isSyllableStart(i)) { + i++; + } + if (i > toUnitIndex) { + return; + } + int iSyllableStart = i; + while (i <= toUnitIndex && !isSyllableEnd(i)) { + i++; + } + if (i > toUnitIndex) { + return; + } + int iSyllableEnd = i; + nextSyllable = new Syllable(features, iSyllableStart, iSyllableEnd); + } + + private boolean isSyllableStart(int index) { + FeatureVector fv = features.getFeatureVector(index); + + return fv.getByteFeature(fiPhone) != fvPhone_0 // not an edge unit + && fv.getByteFeature(fiPhone) != fvPhone_Silence // not silence + && fv.getByteFeature(fiSylStart) == 0 // first segment in syllable + && (!isHalfphone || fv.getByteFeature(fiLR) == fvLR_L); // if halfphone, it's the left half + } + + private boolean isSyllableEnd(int index) { + FeatureVector fv = features.getFeatureVector(index); + + return fv.getByteFeature(fiPhone) != fvPhone_0 // not an edge unit + && fv.getByteFeature(fiPhone) != fvPhone_Silence // not silence + && fv.getByteFeature(fiSylEnd) == 0 // last segment in syllable + && (!isHalfphone || fv.getByteFeature(fiLR) == fvLR_R); // if halfphone, it's the right half + } +} diff --git a/marytts-unitselection/src/main/java/marytts/unitselection/data/TimelineReader.java b/marytts-unitselection/src/main/java/marytts/unitselection/data/TimelineReader.java new file mode 100644 index 0000000000..353c93b3dd --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/unitselection/data/TimelineReader.java @@ -0,0 +1,1304 @@ +/** + * Portions Copyright 2006 DFKI GmbH. + * Portions Copyright 2001 Sun Microsystems, Inc. + * Portions Copyright 1999-2001 Language Technologies Institute, + * Carnegie Mellon University. + * All Rights Reserved. Use is subject to license terms. + * + * Permission is hereby granted, free of charge, to use and distribute + * this software and its documentation without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of this work, and to + * permit persons to whom this work is furnished to do so, subject to + * the following conditions: + * + * 1. The code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * 2. Any modifications must be clearly marked as such. + * 3. Original authors' names are not deleted. + * 4. The authors' names are not used to endorse or promote products + * derived from this software without specific prior written + * permission. + * + * DFKI GMBH AND THE CONTRIBUTORS TO THIS WORK DISCLAIM ALL WARRANTIES WITH + * REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL DFKI GMBH NOR THE + * CONTRIBUTORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL + * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR + * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS + * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF + * THIS SOFTWARE. + */ +package marytts.unitselection.data; + +import java.io.ByteArrayInputStream; +import java.io.DataInput; +import java.io.DataInputStream; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.io.UTFDataFormatException; +import java.nio.BufferUnderflowException; +import java.nio.ByteBuffer; +import java.nio.MappedByteBuffer; +import java.nio.channels.FileChannel; +import java.util.ArrayList; +import java.util.Vector; + +import marytts.exceptions.MaryConfigurationException; +import marytts.util.MaryUtils; +import marytts.util.Pair; +import marytts.util.data.Datagram; +import marytts.util.data.MaryHeader; +import marytts.util.io.StreamUtils; + +/** + * The TimelineReader class provides an interface to read regularly or variably spaced datagrams from a Timeline data file in Mary + * format. + * + * @author sacha, marc + * + */ +public class TimelineReader { + protected MaryHeader maryHdr = null; // The standard Mary header + protected ProcHeader procHdr = null; // The processing info header + + protected Index idx = null; // A global time index for the variable-sized datagrams + + /* Some specific header fields: */ + protected int sampleRate = 0; + protected long numDatagrams = 0; + /** + * The total duration of the timeline data, in samples. This is only computed upon request. + */ + protected long totalDuration = -1; + + protected int datagramsBytePos = 0; + protected int timeIdxBytePos = 0; + + // exactly one of the two following variables will be non-null after load(): + private MappedByteBuffer mappedBB = null; + private FileChannel fileChannel = null; + + /****************/ + /* CONSTRUCTORS */ + /****************/ + + /** + * Construct a timeline from the given file name. + * + * Aiming for the fundamental guarantee: If an instance of this class is created, it is usable. + * + * @param fileName + * The file to read the timeline from. Must be non-null and point to a valid timeline file. + * @throws NullPointerException + * if null argument is given + * @throws MaryConfigurationException + * if no timeline reader can be instantiated from fileName + */ + public TimelineReader(String fileName) throws MaryConfigurationException { + this(fileName, true); + } + + /** + * Construct a timeline from the given file name. + * + * Aiming for the fundamental guarantee: If an instance of this class is created, it is usable. + * + * @param fileName + * The file to read the timeline from. Must be non-null and point to a valid timeline file. + * @param tryMemoryMapping + * if true, will attempt to read audio data via a memory map, and fall back to piecewise reading. If false, will + * immediately go for piecewise reading using a RandomAccessFile. + * @throws NullPointerException + * if null argument is given + * @throws MaryConfigurationException + * if no timeline reader can be instantiated from fileName + */ + public TimelineReader(String fileName, boolean tryMemoryMapping) throws MaryConfigurationException { + if (fileName == null) { + throw new NullPointerException("Filename is null"); + } + try { + load(fileName, tryMemoryMapping); + } catch (Exception e) { + throw new MaryConfigurationException("Cannot load timeline file from " + fileName, e); + } + } + + /** + * Only subclasses can instantiate a TimelineReader object that doesn't call {@link #load(String)}. It is their responsibility + * then to ensure the fundamental guarantee. + */ + protected TimelineReader() { + } + + /** + * Load a timeline from a file. + * + * @param fileName + * The file to read the timeline from. Must be non-null and point to a valid timeline file. + * + * @throws IOException + * if a problem occurs during reading + * @throws BufferUnderflowException + * if a problem occurs during reading + * @throws MaryConfigurationException + * if fileName does not point to a valid timeline file + */ + protected void load(String fileName) throws IOException, BufferUnderflowException, MaryConfigurationException, + NullPointerException { + load(fileName, true); + } + + /** + * Load a timeline from a file. + * + * @param fileName + * The file to read the timeline from. Must be non-null and point to a valid timeline file. + * + * @throws IOException + * if a problem occurs during reading + * @throws BufferUnderflowException + * if a problem occurs during reading + * @throws MaryConfigurationException + * if fileName does not point to a valid timeline file + */ + protected void load(String fileName, boolean tryMemoryMapping) throws IOException, BufferUnderflowException, + MaryConfigurationException, NullPointerException { + assert fileName != null : "filename is null"; + + RandomAccessFile file = new RandomAccessFile(fileName, "r"); + FileChannel fc = file.getChannel(); + // Expect header to be no bigger than 64k bytes + ByteBuffer headerBB = ByteBuffer.allocate(0x10000); + fc.read(headerBB); + headerBB.limit(headerBB.position()); + headerBB.position(0); + + maryHdr = new MaryHeader(headerBB); + if (maryHdr.getType() != MaryHeader.TIMELINE) { + throw new MaryConfigurationException("File is not a valid timeline file."); + } + /* Load the processing info header */ + procHdr = new ProcHeader(headerBB); + + /* Load the timeline dimensions */ + sampleRate = headerBB.getInt(); + numDatagrams = headerBB.getLong(); + if (sampleRate <= 0 || numDatagrams < 0) { + throw new MaryConfigurationException("Illegal values in timeline file."); + } + + /* Load the positions of the various subsequent components */ + datagramsBytePos = (int) headerBB.getLong(); + timeIdxBytePos = (int) headerBB.getLong(); + if (timeIdxBytePos < datagramsBytePos) { + throw new MaryConfigurationException("File seems corrupt: index is expected after data, not before"); + } + + /* Go fetch the time index at the end of the file */ + fc.position(timeIdxBytePos); + ByteBuffer indexBB = ByteBuffer.allocate((int) (fc.size() - timeIdxBytePos)); + fc.read(indexBB); + indexBB.limit(indexBB.position()); + indexBB.position(0); + idx = new Index(indexBB); + + if (tryMemoryMapping) { + // Try if we can use a mapped byte buffer: + try { + mappedBB = fc.map(FileChannel.MapMode.READ_ONLY, datagramsBytePos, timeIdxBytePos - datagramsBytePos); + file.close(); // if map() succeeded, we don't need the file anymore. + } catch (IOException ome) { + MaryUtils.getLogger("Timeline").warn( + "Cannot use memory mapping for timeline file '" + fileName + "' -- falling back to piecewise reading"); + } + } + if (!tryMemoryMapping || mappedBB == null) { // use piecewise reading + fileChannel = fc; + assert fileChannel != null; + // and leave file open + } + + // postconditions: + assert idx != null; + assert procHdr != null; + assert fileChannel == null && mappedBB != null || fileChannel != null && mappedBB == null; + } + + /** + * Return the content of the processing header as a String. + * + * @return a non-null string representing the proc header. + */ + public String getProcHeaderContents() { + return procHdr.getString(); + } + + /** + * Returns the number of datagrams in the timeline. + * + * @return the (non-negative) number of datagrams, as a long. + */ + public long getNumDatagrams() { + assert numDatagrams >= 0; + return numDatagrams; + } + + /** + * Returns the position of the datagram zone in the original file. + * + * @return the byte position of the datagram zone. + */ + protected long getDatagramsBytePos() { + return datagramsBytePos; + } + + /** + * Returns the timeline's sample rate. + * + * @return the sample rate as a positive integer. + */ + public int getSampleRate() { + assert sampleRate > 0; + return sampleRate; + } + + /** + * Return the total duration of all data in this timeline. Implementation note: this is an expensive operation that should not + * be used in production. + * + * @return a non-negative long representing the accumulated duration of all datagrams. + * @throws MaryConfigurationException + * if the duration cannot be obtained. + */ + public long getTotalDuration() throws MaryConfigurationException { + if (totalDuration == -1) { + computeTotalDuration(); + } + assert totalDuration >= 0; + return totalDuration; + } + + /** + * Compute the total duration of a timeline. This is an expensive method, since it goes through all datagrams to compute this + * duration. It should not normally be used in production. + * + * @throws MaryConfigurationException + * if the duration could not be computed. + */ + protected void computeTotalDuration() throws MaryConfigurationException { + long time = 0; + long nRead = 0; + boolean haveReadAll = false; + try { + Pair p = getByteBufferAtTime(0); + ByteBuffer bb = p.getFirst(); + assert p.getSecond() == 0; + while (!haveReadAll) { + Datagram dat = getNextDatagram(bb); + if (dat == null) { + // we may have reached the end of the current byte buffer... try reading another: + p = getByteBufferAtTime(time); + bb = p.getFirst(); + assert p.getSecond() == time; + dat = getNextDatagram(bb); + if (dat == null) { // no, indeed we cannot read any more + break; // abort, we could not read all + } + } + assert dat != null; + time += dat.getDuration(); // duration in timeline sample rate + nRead++; // number of datagrams read + if (nRead == numDatagrams) { + haveReadAll = true; + } + } + } catch (Exception e) { + throw new MaryConfigurationException("Could not compute total duration", e); + } + if (!haveReadAll) { + throw new MaryConfigurationException("Could not read all datagrams to compute total duration"); + } + totalDuration = time; + } + + /** + * The index object. + * + * @return the non-null index object. + */ + public Index getIndex() { + assert idx != null; + return idx; + } + + // Helper methods + + /** + * Scales a discrete time to the timeline's sample rate. + * + * @param reqSampleRate + * the externally given sample rate. + * @param targetTimeInSamples + * a discrete time, with respect to the externally given sample rate. + * + * @return a discrete time, in samples with respect to the timeline's sample rate. + */ + protected long scaleTime(int reqSampleRate, long targetTimeInSamples) { + if (reqSampleRate == sampleRate) + return (targetTimeInSamples); + /* else */return ((long) Math.round((double) (reqSampleRate) * (double) (targetTimeInSamples) / (double) (sampleRate))); + } + + /** + * Unscales a discrete time from the timeline's sample rate. + * + * @param reqSampleRate + * the externally given sample rate. + * @param timelineTimeInSamples + * a discrete time, with respect to the timeline sample rate. + * + * @return a discrete time, in samples with respect to the externally given sample rate. + */ + protected long unScaleTime(int reqSampleRate, long timelineTimeInSamples) { + if (reqSampleRate == sampleRate) + return (timelineTimeInSamples); + /* else */return ((long) Math.round((double) (sampleRate) * (double) (timelineTimeInSamples) / (double) (reqSampleRate))); + } + + /******************/ + /* DATA ACCESSORS */ + /******************/ + + /** + * Skip the upcoming datagram at the current position of the byte buffer. + * + * @return the duration of the datagram we skipped + * @throws IOException + * if we cannot skip another datagram because we have reached the end of the byte buffer + */ + protected long skipNextDatagram(ByteBuffer bb) throws IOException { + long datagramDuration = bb.getLong(); + int datagramSize = bb.getInt(); + if (bb.position() + datagramSize > bb.limit()) { + throw new IOException("cannot skip datagram: it is not fully contained in byte buffer"); + } + bb.position(bb.position() + datagramSize); + return datagramDuration; + } + + /** + * Read and return the upcoming datagram from the given byte buffer. Subclasses should override this method to create + * subclasses of Datagram. + * + * @param bb + * the timeline byte buffer to read from + * + * @return the current datagram, or null if EOF was encountered + */ + protected Datagram getNextDatagram(ByteBuffer bb) { + assert bb != null; + // If the end of the datagram zone is reached, refuse to read + if (bb.position() == bb.limit()) { + return null; + } + // Else, read the datagram from the file + try { + return new Datagram(bb); + } catch (IOException ioe) { + return null; + } + } + + /** + * Hop the datagrams in the given byte buffer until the one which begins at or contains the desired time (time is in samples; + * the sample rate is assumed to be that of the timeline). + * + * @param bb + * the timeline byte buffer to use. Must not be null. + * @param currentTimeInSamples + * the time position corresponding to the current position of the byte buffer. Must not be negative. + * @param targetTimeInSamples + * the time location to reach. Must not be less than currentTimeInSamples + * + * @return the actual time at which we end up after hopping. This is less than or equal to targetTimeInSamples, never greater + * than it. + * @throws IOException + * if there is a problem skipping the datagrams + * @throws IllegalArgumentException + * if targetTimeInSamples is less than currentTimeInSamples + */ + protected long hopToTime(ByteBuffer bb, long currentTimeInSamples, long targetTimeInSamples) throws IOException, + IllegalArgumentException { + assert bb != null; + assert currentTimeInSamples >= 0; + assert targetTimeInSamples >= currentTimeInSamples : "Cannot hop back from time " + currentTimeInSamples + " to time " + + targetTimeInSamples; + + /* + * If the current time position is the requested time do nothing, you are already at the right position + */ + if (currentTimeInSamples == targetTimeInSamples) { + return currentTimeInSamples; + } + /* Else hop: */ + int byteBefore = bb.position(); + long timeBefore = currentTimeInSamples; + /* Hop until the datagram which comes just after the requested time */ + while (currentTimeInSamples <= targetTimeInSamples) { // Stop after the requested time, we will step back + // to the correct time in case of equality + timeBefore = currentTimeInSamples; + byteBefore = bb.position(); + long skippedDuration = skipNextDatagram(bb); + currentTimeInSamples += skippedDuration; + } + /* Do one step back so that the pointed datagram contains the requested time */ + bb.position(byteBefore); + return timeBefore; + } + + /** + * This method produces a new byte buffer whose current position represents the requested positionInFile. It cannot be assumed + * that a call to byteBuffer.position() produces any meaningful values. The byte buffer may represent only a part of the + * available data; however, at least one datagram can be read from the byte buffer. If no further data can be read from it, a + * new byte buffer must be obtained by calling this method again with a new target time. + * + * @param targetTimeInSamples + * the time position in the file which should be accessed as a byte buffer, in samples. Must be non-negative and + * less than the total duration of the timeline. + * @return a pair representing the byte buffer from which to read, and the exact time corresponding to the current position of + * the byte buffer. The position as such is not meaningful; the time is guaranteed to be less than or equal to + * targetTimeInSamples. + * @throws IOException + * , BufferUnderflowException if no byte buffer can be obtained for the requested time. + */ + protected Pair getByteBufferAtTime(long targetTimeInSamples) throws IOException, BufferUnderflowException { + if (mappedBB != null) { + return getMappedByteBufferAtTime(targetTimeInSamples); + } else { + return loadByteBufferAtTime(targetTimeInSamples); + } + } + + protected Pair getMappedByteBufferAtTime(long targetTimeInSamples) throws IllegalArgumentException, + IOException { + assert mappedBB != null; + /* Seek for the time index which comes just before the requested time */ + IdxField idxFieldBefore = idx.getIdxFieldBefore(targetTimeInSamples); + long time = idxFieldBefore.timePtr; + int bytePos = (int) (idxFieldBefore.bytePtr - datagramsBytePos); + ByteBuffer bb = mappedBB.duplicate(); + bb.position(bytePos); + time = hopToTime(bb, time, targetTimeInSamples); + return new Pair(bb, time); + } + + protected Pair loadByteBufferAtTime(long targetTimeInSamples) throws IOException { + assert fileChannel != null; + // we must load a chunk of data from the FileChannel + int bufSize = 0x10000; // 64 kB + /* Seek for the time index which comes just before the requested time */ + IdxField idxFieldBefore = idx.getIdxFieldBefore(targetTimeInSamples); + long time = idxFieldBefore.timePtr; + long bytePos = idxFieldBefore.bytePtr; + if (bytePos + bufSize > timeIdxBytePos) { // must not read index data as datagrams + bufSize = (int) (timeIdxBytePos - bytePos); + } + ByteBuffer bb = loadByteBuffer(bytePos, bufSize); + + while (true) { + if (!canReadDatagramHeader(bb)) { + bb = loadByteBuffer(bytePos, bufSize); + assert canReadDatagramHeader(bb); + } + int posBefore = bb.position(); + Datagram d = new Datagram(bb, false); + if (time + d.getDuration() > targetTimeInSamples) { // d is our datagram + bb.position(posBefore); + int datagramNumBytes = Datagram.NUM_HEADER_BYTES + d.getLength(); + // need to make sure we return a byte buffer from which d can be read + if (!canReadAmount(bb, datagramNumBytes)) { + bb = loadByteBuffer(bytePos, Math.max(datagramNumBytes, bufSize)); + } + assert canReadAmount(bb, datagramNumBytes); + break; + } else { + // keep on skipping + time += d.getDuration(); + if (canReadAmount(bb, d.getLength())) { + bb.position(bb.position() + d.getLength()); + } else { + bytePos += bb.position(); + bytePos += d.getLength(); + bb = loadByteBuffer(bytePos, bufSize); + } + } + } + return new Pair(bb, time); + } + + /** + * @param bytePos + * position in fileChannel from which to load the byte buffer + * @param bufSize + * size of the byte buffer + * @return the byte buffer, loaded and set such that limit is bufSize and position is 0 + * @throws IOException + * if the data cannot be read from fileChannel + */ + private ByteBuffer loadByteBuffer(long bytePos, int bufSize) throws IOException { + ByteBuffer bb = ByteBuffer.allocate(bufSize); + fileChannel.read(bb, bytePos); // this will block if another thread is currently reading from fileChannel + bb.limit(bb.position()); + bb.position(0); + return bb; + } + + private boolean canReadDatagramHeader(ByteBuffer bb) { + return canReadAmount(bb, Datagram.NUM_HEADER_BYTES); + } + + private boolean canReadAmount(ByteBuffer bb, int amount) { + return bb.limit() - bb.position() >= amount; + } + + /** + * Get a single datagram from a particular time location, given in the timeline's sampling rate. + * + * @param targetTimeInSamples + * the requested position, in samples. Must be non-negative and less than the total duration of the timeline. + * + * @return the datagram starting at or overlapping the given time, or null if end-of-file was encountered + * @throws IOException + * , BufferUnderflowException if no datagram could be created from the data at the given time. + */ + public Datagram getDatagram(long targetTimeInSamples) throws IOException { + Pair p = getByteBufferAtTime(targetTimeInSamples); + ByteBuffer bb = p.getFirst(); + return getNextDatagram(bb); + } + + /** + * Get a single datagram from a particular time location. + * + * @param targetTimeInSamples + * the requested position, in samples. Must be non-negative and less than the total duration of the timeline. + * @param reqSampleRate + * the sample rate for the requested times. + * + * @return the datagram starting at or overlapping the given time, or null if end-of-file was encountered + * @throws IOException + * if no datagram could be created from the data at the given time. + */ + public Datagram getDatagram(long targetTimeInSamples, int reqSampleRate) throws IOException { + /* + * Resample the requested time location, in case the sample times are different between the request and the timeline + */ + long scaledTargetTime = scaleTime(reqSampleRate, targetTimeInSamples); + Datagram dat = getDatagram(scaledTargetTime); + if (dat == null) + return null; + if (reqSampleRate != sampleRate) + dat.setDuration(unScaleTime(reqSampleRate, dat.getDuration())); // => Don't forget to stay time-consistent! + return dat; + } + + /** + * Get the datagrams spanning a particular time range from a particular time location, and return the time offset between the + * time request and the actual location of the first returned datagram. Irrespective of the values of nDatagrams and + * timeSpanInSamples, at least one datagram is always returned. + * + * @param targetTimeInSamples + * the requested position, in samples. Must be non-negative and less than the total duration of the timeline. + * @param nDatagrams + * the number of datagrams to read. Ignored if timeSpanInSamples is positive. + * @param timeSpanInSamples + * the requested time span, in samples. If positive, then datagrams are selected by the given time span. + * @param reqSampleRate + * the sample rate for the requested and returned times. Must be positive. + * @param returnOffset + * an optional output field. If it is not null, then after the call it must have length of at least 1, and the + * first array field will contain the time difference, in samples, between the time request and the actual + * beginning of the first datagram. + * + * @return an array of datagrams containing at least one datagram. If less than the requested amount of datagrams can be read, + * the number of datagrams that can be read is returned. + * @throws IllegalArgumentException + * if targetTimeInSamples is negative, or if a returnOffset of length 0 is given. + * @throws IOException + * if no data can be read at the given target time + */ + private Datagram[] getDatagrams(long targetTimeInSamples, int nDatagrams, long timeSpanInSamples, int reqSampleRate, + long[] returnOffset) throws IllegalArgumentException, IOException { + /* Check the input arguments */ + if (targetTimeInSamples < 0) { + throw new IllegalArgumentException("Can't get a datagram from a negative time position (given time position was [" + + targetTimeInSamples + "])."); + } + if (reqSampleRate <= 0) { + throw new IllegalArgumentException("sample rate must be positive, but is " + reqSampleRate); + } + // Get the datagrams by number or by time span? + boolean byNumber; + if (timeSpanInSamples > 0) { + byNumber = false; + } else { + byNumber = true; + if (nDatagrams <= 0) { + nDatagrams = 1; // return at least one datagram + } + } + + /* + * Resample the requested time location, in case the sample times are different between the request and the timeline + */ + long scaledTargetTime = scaleTime(reqSampleRate, targetTimeInSamples); + + Pair p = getByteBufferAtTime(scaledTargetTime); + ByteBuffer bb = p.getFirst(); + long time = p.getSecond(); + if (returnOffset != null) { // return offset between target and actual start time + if (returnOffset.length == 0) { + throw new IllegalArgumentException("If returnOffset is given, it must have length of at least 1"); + } + returnOffset[0] = unScaleTime(reqSampleRate, (scaledTargetTime - time)); + } + + ArrayList datagrams = new ArrayList(byNumber ? nDatagrams : 10); + // endTime is stop criterion if reading by time scale: + long endTime = byNumber ? -1 : scaleTime(reqSampleRate, (targetTimeInSamples + timeSpanInSamples)); + int nRead = 0; + boolean haveReadAll = false; + while (!haveReadAll) { + Datagram dat = getNextDatagram(bb); + if (dat == null) { + // we may have reached the end of the current byte buffer... try reading another: + try { + p = getByteBufferAtTime(time); + } catch (Exception ioe) { + // cannot get another byte buffer -- stop reading. + break; + } + bb = p.getFirst(); + dat = getNextDatagram(bb); + if (dat == null) { // no, indeed we cannot read any more + break; // abort, we could not read all + } + } + assert dat != null; + time += dat.getDuration(); // duration in timeline sample rate + nRead++; // number of datagrams read + if (reqSampleRate != sampleRate) { + dat.setDuration(unScaleTime(reqSampleRate, dat.getDuration())); // convert duration into reqSampleRate + } + datagrams.add(dat); + if (byNumber && nRead == nDatagrams || !byNumber && time >= endTime) { + haveReadAll = true; + } + } + return (Datagram[]) datagrams.toArray(new Datagram[0]); + } + + // ///////////////////// Convenience methods: variants of getDatagrams() /////////////////////// + + // ///////////////////// by time span //////////////////////////// + + /** + * Get the datagrams spanning a particular time range from a particular time location, and return the time offset between the + * time request and the actual location of the first returned datagram. Irrespective of the value of timeSpanInSamples, at + * least one datagram is always returned. + * + * @param targetTimeInSamples + * the requested position, in samples. Must be non-negative and less than the total duration of the timeline. + * @param timeSpanInSamples + * the requested time span, in samples. If positive, then datagrams are selected by the given time span. + * @param reqSampleRate + * the sample rate for the requested and returned times. Must be positive. + * @param returnOffset + * an optional output field. If it is not null, then after the call it must have length of at least 1, and the + * first array field will contain the time difference, in samples, between the time request and the actual + * beginning of the first datagram. + * + * @return an array of datagrams containing at least one datagram. If less than the requested amount of datagrams can be read, + * the number of datagrams that can be read is returned. + * @throws IllegalArgumentException + * if targetTimeInSamples is negative, or if a returnOffset of length 0 is given. + * @throws IOException + * , BufferUnderflowException if no data can be read at the given target time + */ + public Datagram[] getDatagrams(long targetTimeInSamples, long timeSpanInSamples, int reqSampleRate, long[] returnOffset) + throws IOException { + return getDatagrams(targetTimeInSamples, -1, timeSpanInSamples, reqSampleRate, returnOffset); + } + + /** + * Get the datagrams spanning a particular time range from a particular time location. Irrespective of the value of + * timeSpanInSamples, at least one datagram is always returned. + * + * @param targetTimeInSamples + * the requested position, in samples. Must be non-negative and less than the total duration of the timeline. + * @param timeSpanInSamples + * the requested time span, in samples. If positive, then datagrams are selected by the given time span. + * @param reqSampleRate + * the sample rate for the requested and returned times. Must be positive. + * + * @return an array of datagrams containing at least one datagram. If less than the requested amount of datagrams can be read, + * the number of datagrams that can be read is returned. + * @throws IllegalArgumentException + * if targetTimeInSamples is negative, or if a returnOffset of length 0 is given. + * @throws IOException + * if no data can be read at the given target time + */ + public Datagram[] getDatagrams(long targetTimeInSamples, long timeSpanInSamples, int reqSampleRate) throws IOException { + return getDatagrams(targetTimeInSamples, timeSpanInSamples, reqSampleRate, null); + } + + /** + * Get a given number of datagrams from a particular time location. + * + * @param targetTimeInSamples + * the requested position, in samples. Must be non-negative and less than the total duration of the timeline. + * @param number + * the number of datagrams to read. Even if this is <= 0, at least one datagram is always returned. + * @param reqSampleRate + * the sample rate for the requested and returned times. Must be positive. + * + * @return an array of datagrams containing at least one datagram. If less than the requested amount of datagrams can be read, + * the number of datagrams that can be read is returned. + * @throws IllegalArgumentException + * if targetTimeInSamples is negative, or if a returnOffset of length 0 is given. + * @throws IOException + * if no data can be read at the given target time + */ + public Datagram[] getDatagrams(long targetTimeInSamples, long timeSpanInSamples) throws IOException { + return getDatagrams(targetTimeInSamples, timeSpanInSamples, sampleRate, null); + } + + // ///////////////////// by number of datagrams //////////////////////////// + + /** + * Get a given number of datagrams from a particular time location, and return the time offset between the time request and + * the actual location of the first returned datagram. + * + * @param targetTimeInSamples + * the requested position, in samples. Must be non-negative and less than the total duration of the timeline. + * @param number + * the number of datagrams to read. Even if this is <= 0, at least one datagram is always returned. + * @param reqSampleRate + * the sample rate for the requested and returned times. Must be positive. + * @param returnOffset + * an optional output field. If it is not null, then after the call it must have length of at least 1, and the + * first array field will contain the time difference, in samples, between the time request and the actual + * beginning of the first datagram. + * + * @return an array of datagrams containing at least one datagram. If less than the requested amount of datagrams can be read, + * the number of datagrams that can be read is returned. + * @throws IllegalArgumentException + * if targetTimeInSamples is negative, or if a returnOffset of length 0 is given. + * @throws IOException + * if no data can be read at the given target time + */ + public Datagram[] getDatagrams(long targetTimeInSamples, int number, int reqSampleRate, long[] returnOffset) + throws IOException { + return getDatagrams(targetTimeInSamples, number, -1, reqSampleRate, returnOffset); + } + + // ///////////////////// by unit //////////////////////////// + + /** + * Get the datagrams spanning a particular unit, and return the time offset between the unit request and the actual location + * of the first returned datagram. Irrespective of the unit duration, at least one datagram is always returned. + * + * @param unit + * The requested speech unit, containing its own position and duration. + * @param reqSampleRate + * the sample rate for the requested and returned times. Must be positive. + * @param returnOffset + * an optional output field. If it is not null, then after the call it must have length of at least 1, and the + * first array field will contain the time difference, in samples, between the time request and the actual + * beginning of the first datagram. + * + * @return an array of datagrams containing at least one datagram. If less than the requested amount of datagrams can be read, + * the number of datagrams that can be read is returned. + * @throws IllegalArgumentException + * if targetTimeInSamples is negative, or if a returnOffset of length 0 is given. + * @throws IOException + * if no data can be read at the given target time + */ + public Datagram[] getDatagrams(Unit unit, int reqSampleRate, long[] returnOffset) throws IOException { + return getDatagrams(unit.startTime, (long) (unit.duration), reqSampleRate, returnOffset); + } + + /** + * Get the datagrams spanning a particular unit. Irrespective of the unit duration, at least one datagram is always returned. + * + * @param unit + * The requested speech unit, containing its own position and duration. + * @param reqSampleRate + * the sample rate for the requested and returned times. Must be positive. + * + * @return an array of datagrams containing at least one datagram. If less than the requested amount of datagrams can be read, + * the number of datagrams that can be read is returned. + * @throws IllegalArgumentException + * if targetTimeInSamples is negative, or if a returnOffset of length 0 is given. + * @throws IOException + * if no data can be read at the given target time + */ + public Datagram[] getDatagrams(Unit unit, int reqSampleRate) throws IOException { + return getDatagrams(unit, reqSampleRate, null); + } + + /*****************************************/ + /* HELPER CLASSES */ + /*****************************************/ + + /** + * Simple helper class to read the index part of a timeline file. The index points to datagrams at or before a certain point + * in time. + * + * Note: If no datagram starts at the exact index time, it makes sense to point to the previous datagram rather than the + * following one. + * + * If one would store the location of the datagram which comes just after the index position (the currently tested datagram), + * there would be a possibility that a particular time request falls between the index and the datagram: + * + * time axis ---------------------------------> INDEX <-- REQUEST | ---------------> DATAGRAM + * + * This would require a subsequent backwards time hopping, which is impossible because the datagrams are a singly linked list. + * + * By registering the location of the previous datagram, any time request will find an index which points to a datagram + * falling BEFORE or ON the index location: + * + * time axis ---------------------------------> INDEX <-- REQUEST | DATAGRAM <--- + * + * Thus, forward hopping is always possible and the requested time can always be reached. + * + * @author sacha + */ + public static class Index { + private int idxInterval = 0; // The fixed time interval (in samples) separating two index fields. + + /** + * For index field i, bytePtrs[i] is the position in bytes, from the beginning of the file, of the datagram coming on or + * just before that index field. + */ + private long[] bytePtrs; + + /** + * For index field i, timePtrs[i] is the time position in samples of the datagram coming on or just before that index + * field. + */ + private long[] timePtrs; + + /****************/ + /* CONSTRUCTORS */ + /****************/ + + /** + * Construct an index from a data input stream or random access file. Fundamental guarantee: Once created, the index is + * guaranteed to contain a positive index interval and monotonously rising byte and time pointers. + * + * @param bb + * byte buffer from which to read the index. Must not be null, and read position must be at start of index. + * @throws IOException + * if there is a problem reading. + * @throws MaryConfigurationException + * if the index is not well-formed. + */ + private Index(DataInput raf) throws IOException, MaryConfigurationException { + assert raf != null : "null argument"; + load(raf); + } + + /** + * Construct an index from a byte buffer. Fundamental guarantee: Once created, the index is guaranteed to contain a + * positive index interval and monotonously rising byte and time pointers. + * + * @param rafIn + * data input from which to read the index. Must not be null, and read position must be at start of index. + * @throws BufferUnderflowException + * if there is a problem reading. + * @throws MaryConfigurationException + * if the index is not well-formed. + */ + private Index(ByteBuffer bb) throws BufferUnderflowException, MaryConfigurationException { + assert bb != null : "null argument"; + load(bb); + } + + /** + * Constructor which builds a new index with a specific index interval and a given sample rate. Fundamental guarantee: + * Once created, the index is guaranteed to contain a positive index interval and monotonously rising byte and time + * pointers. + * + * @param idxInterval + * the index interval, in samples. Must be a positive number. + * @param indexFields + * the actual index data. Must not be null. + * @throws IllegalArgumentException + * if the index data given is not well-formed. + * @throws NullPointerException + * if indexFields are null. + */ + public Index(int idxInterval, Vector indexFields) throws IllegalArgumentException, NullPointerException { + if (idxInterval <= 0) { + throw new IllegalArgumentException("got index interval <= 0"); + } + if (indexFields == null) { + throw new NullPointerException("null argument"); + } + this.idxInterval = idxInterval; + bytePtrs = new long[indexFields.size()]; + timePtrs = new long[indexFields.size()]; + for (int i = 0; i < bytePtrs.length; i++) { + IdxField f = indexFields.get(i); + bytePtrs[i] = f.bytePtr; + timePtrs[i] = f.timePtr; + if (i > 0) { + if (bytePtrs[i] < bytePtrs[i - 1] || timePtrs[i] < timePtrs[i - 1]) { + throw new IllegalArgumentException( + "Pointer positions in index fields must be strictly monotonously rising"); + } + } + } + } + + /*****************/ + /* I/O METHODS */ + /*****************/ + + /** + * Method which loads an index from a data input (random access file or data input stream). + * + * @param rafIn + * data input from which to read the index. Must not be null, and read position must be at start of index. + * @throws IOException + * if there is a problem reading. + * @throws MaryConfigurationException + * if the index is not well-formed. + */ + public void load(DataInput rafIn) throws IOException, MaryConfigurationException { + int numIdx = rafIn.readInt(); + idxInterval = rafIn.readInt(); + if (idxInterval <= 0) { + throw new MaryConfigurationException("read negative index interval -- file seems corrupt"); + } + + bytePtrs = new long[numIdx]; + timePtrs = new long[numIdx]; + int numBytesToRead = 16 * numIdx + 16; // 2*8 bytes for each index field + 16 for prevBytePos and prevTimePos + + byte[] data = new byte[numBytesToRead]; + rafIn.readFully(data); + DataInput bufIn = new DataInputStream(new ByteArrayInputStream(data)); + + for (int i = 0; i < numIdx; i++) { + bytePtrs[i] = bufIn.readLong(); + timePtrs[i] = bufIn.readLong(); + if (i > 0) { + if (bytePtrs[i] < bytePtrs[i - 1] || timePtrs[i] < timePtrs[i - 1]) { + throw new MaryConfigurationException( + "File seems corrupt: Pointer positions in index fields are not strictly monotonously rising"); + } + } + } + /* Obsolete: Read the "last datagram" memory */ + /* prevBytePos = */bufIn.readLong(); + /* prevTimePos = */bufIn.readLong(); + } + + /** + * Method which loads an index from a byte buffer. + * + * @param bb + * byte buffer from which to read the index. Must not be null, and read position must be at start of index. + * @throws BufferUnderflowException + * if there is a problem reading. + * @throws MaryConfigurationException + * if the index is not well-formed. + */ + private void load(ByteBuffer bb) throws BufferUnderflowException, MaryConfigurationException { + int numIdx = bb.getInt(); + idxInterval = bb.getInt(); + if (idxInterval <= 0) { + throw new MaryConfigurationException("read negative index interval -- file seems corrupt"); + } + + bytePtrs = new long[numIdx]; + timePtrs = new long[numIdx]; + + for (int i = 0; i < numIdx; i++) { + bytePtrs[i] = bb.getLong(); + timePtrs[i] = bb.getLong(); + if (i > 0) { + if (bytePtrs[i] < bytePtrs[i - 1] || timePtrs[i] < timePtrs[i - 1]) { + throw new MaryConfigurationException( + "File seems corrupt: Pointer positions in index fields are not strictly monotonously rising"); + } + } + } + /* Obsolete: Read the "last datagram" memory */ + /* prevBytePos = */bb.getLong(); + /* prevTimePos = */bb.getLong(); + } + + /** + * Method which writes an index to a RandomAccessFile + * */ + public long dump(RandomAccessFile rafIn) throws IOException { + long nBytes = 0; + int numIdx = getNumIdx(); + rafIn.writeInt(numIdx); + nBytes += 4; + rafIn.writeInt(idxInterval); + nBytes += 4; + for (int i = 0; i < numIdx; i++) { + rafIn.writeLong(bytePtrs[i]); + nBytes += 8; + rafIn.writeLong(timePtrs[i]); + nBytes += 8; + } + // Obsolete, keep only for file format compatibility: + // Register the "last datagram" memory as an additional field + // rafIn.writeLong(prevBytePos); + // rafIn.writeLong(prevTimePos); + rafIn.writeLong(0l); + rafIn.writeLong(0l); + nBytes += 16l; + + return nBytes; + } + + /** + * Method which writes an index to stdout + * */ + public void print() { + System.out.println(""); + int numIdx = getNumIdx(); + System.out.println("interval = " + idxInterval); + System.out.println("numIdx = " + numIdx); + for (int i = 0; i < numIdx; i++) { + System.out.println("( " + bytePtrs[i] + " , " + timePtrs[i] + " )"); + } + /* Obsolete: Register the "last datagram" memory as an additional field */ + // System.out.println( "Last datagram: " + // + "( " + prevBytePos + " , " + prevTimePos + " )" ); + System.out.println(""); + } + + /*****************/ + /* ACCESSORS */ + /*****************/ + /** + * The number of index entries. + */ + public int getNumIdx() { + return bytePtrs.length; + } + + /** + * The interval, in samples, between two index entries. + * + * @return + */ + public int getIdxInterval() { + return idxInterval; + } + + public IdxField getIdxField(int i) { + if (i < 0) { + throw new IndexOutOfBoundsException("Negative index."); + } + if (i >= bytePtrs.length) { + throw new IndexOutOfBoundsException("Requested index no. " + i + ", but highest is " + bytePtrs.length); + } + return new IdxField(bytePtrs[i], timePtrs[i]); + } + + /*****************/ + /* OTHER METHODS */ + /*****************/ + + /** + * Returns the index field that comes immediately before or straight on the requested time. + * + * @param timePosition + * the non-negative time + * @return an index field representing the index position just before or straight on the requested time. + * @throws IllegalArgumentException + * if the given timePosition is negtive + */ + public IdxField getIdxFieldBefore(long timePosition) { + if (timePosition < 0) { + throw new IllegalArgumentException("Negative time given"); + } + int index = (int) (timePosition / idxInterval); /* + * <= This is an integer division between two longs, implying a + * flooring operation on the decimal result. + */ + // System.out.println( "TIMEPOS=" + timePosition + " IDXINT=" + idxInterval + " IDX=" + idx ); + // System.out.flush(); + if (index < 0) { + throw new RuntimeException("Negative index field: [" + index + "] encountered when getting index before time=[" + + timePosition + "] (idxInterval=[" + idxInterval + "])."); + } + if (index >= bytePtrs.length) { + index = bytePtrs.length - 1; // <= Protection against ArrayIndexOutOfBounds exception due to "time out of bounds" + } + return new IdxField(bytePtrs[index], timePtrs[index]); + } + } + + /** + * Simple helper class to read the index fields in a timeline. + * + * @author sacha + * + */ + public static class IdxField { + // TODO: rethink if these should be public fields or if we should add accessors. + public long bytePtr = 0; + public long timePtr = 0; + + public IdxField() { + bytePtr = 0; + timePtr = 0; + } + + public IdxField(long setBytePtr, long setTimePtr) { + bytePtr = setBytePtr; + timePtr = setTimePtr; + } + } + + /** + * + * Simple helper class to load the processing header. + * + * @author sacha + * + */ + public static class ProcHeader { + + private String procHeader = null; + + /****************/ + /* CONSTRUCTORS */ + /****************/ + + /** + * Constructor which loads the procHeader from a RandomAccessFile. Fundamental guarantee: after creation, the ProcHeader + * object has a non-null (but possibly empty) string content. + * + * @param raf + * input from which to load the processing header. Must not be null and must be positioned so that a processing + * header can be read from it. + * + * @throws IOException + * if no proc header can be read at the current position. + */ + private ProcHeader(RandomAccessFile raf) throws IOException { + loadProcHeader(raf); + } + + /** + * Constructor which loads the procHeader from a RandomAccessFile Fundamental guarantee: after creation, the ProcHeader + * object has a non-null (but possibly empty) string content. + * + * @param raf + * input from which to load the processing header. Must not be null and must be positioned so that a processing + * header can be read from it. + * + * @throws BufferUnderflowException + * , UTFDataFormatException if no proc header can be read at the current position. + */ + private ProcHeader(ByteBuffer bb) throws BufferUnderflowException, UTFDataFormatException { + loadProcHeader(bb); + } + + /** + * Constructor which makes the procHeader from a String. Fundamental guarantee: after creation, the ProcHeader object has + * a non-null (but possibly empty) string content. + * + * @param procStr + * a non-null string representing the contents of the ProcHeader. + * @throws NullPointerException + * if procStr is null + * */ + public ProcHeader(String procStr) { + if (procStr == null) { + throw new NullPointerException("null argument"); + } + procHeader = procStr; + } + + /****************/ + /* ACCESSORS */ + /****************/ + + /** + * Return the string length of the proc header. + * + * @return a non-negative int representling the string length of the proc header. + */ + public int getCharSize() { + assert procHeader != null; + return procHeader.length(); + } + + /** + * Get the string content of the proc header. + * + * @return a non-null string representing the string content of the proc header. + */ + public String getString() { + assert procHeader != null; + return procHeader; + } + + /*****************/ + /* I/O METHODS */ + /*****************/ + + /** + * Method which loads the header from a RandomAccessFile. + * + * @param rafIn + * file to read from, must not be null. + * @throws IOException + * if no proc header can be read at the current position. + */ + private void loadProcHeader(RandomAccessFile rafIn) throws IOException { + assert rafIn != null : "null argument"; + procHeader = rafIn.readUTF(); + assert procHeader != null; + } + + /** + * Method which loads the header from a byte buffer. + * + * @param bb + * byte buffer to read from, must not be null. + * @throws BufferUnderflowException + * , UTFDataFormatException if no proc header can be read at the current position. + */ + private void loadProcHeader(ByteBuffer bb) throws BufferUnderflowException, UTFDataFormatException { + procHeader = StreamUtils.readUTF(bb); + assert procHeader != null; + } + + /** + * Method which writes the proc header to a RandomAccessFile. + * + * @return the number of written bytes. + * */ + public long dump(RandomAccessFile rafIn) throws IOException { + long before = rafIn.getFilePointer(); + rafIn.writeUTF(procHeader); + long after = rafIn.getFilePointer(); + return after - before; + } + } + +} \ No newline at end of file diff --git a/marytts-unitselection/src/main/java/marytts/unitselection/data/Unit.java b/marytts-unitselection/src/main/java/marytts/unitselection/data/Unit.java new file mode 100644 index 0000000000..64c7dae6b3 --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/unitselection/data/Unit.java @@ -0,0 +1,85 @@ +/** + * Copyright 2006 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.unitselection.data; + +/** + * Representation of a unit from a unit database. This gives access to everything that is known about a given unit, including all + * sorts of features and the actual audio data. + * + * @author Marc Schröder + * + */ +public class Unit { + + /** + * Unit start time, expressed in samples. To convert into time, divide by UnitFileReader.getSampleRate(). + */ + public final long startTime; + + /** + * Unit duration, expressed in samples. To convert into time, divide by UnitFileReader.getSampleRate(). + */ + public final int duration; + + /** + * Index position of this unit in the unit file. + */ + public final int index; + + public Unit(long startTime, int duration, int index) { + this.startTime = startTime; + this.duration = duration; + this.index = index; + } + + /** + * Determine whether the unit is an "edge" unit, i.e. a unit marking the start or the end of an utterance. + * + * @param i + * The index of the considered unit. + * @return true if the unit is an edge unit, false otherwise + */ + public boolean isEdgeUnit() { + return duration == -1; + } + + public String toString() { + return "unit " + index + " start: " + startTime + ", duration: " + duration; + } + + /** + * inspired by http://www.artima.com/lejava/articles/equality.html + */ + @Override + public boolean equals(Object other) { + boolean result = false; + if (other instanceof Unit) { + Unit that = (Unit) other; + result = (this.index == that.index); + } + return result; + } + + @Override + public int hashCode() { + return this.index; + } + +} diff --git a/marytts-unitselection/src/main/java/marytts/unitselection/data/UnitDatabase.java b/marytts-unitselection/src/main/java/marytts/unitselection/data/UnitDatabase.java new file mode 100644 index 0000000000..46652588f7 --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/unitselection/data/UnitDatabase.java @@ -0,0 +1,195 @@ +/** + * Copyright 2006 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.unitselection.data; + +import java.util.ArrayList; +import java.util.List; + +import marytts.cart.CART; +import marytts.unitselection.select.JoinCostFunction; +import marytts.unitselection.select.StatisticalCostFunction; +import marytts.features.Target; +import marytts.unitselection.select.TargetCostFunction; +import marytts.unitselection.select.viterbi.ViterbiCandidate; +import marytts.util.MaryUtils; +import marytts.util.data.Datagram; +import marytts.util.dom.DomUtils; + +import org.apache.log4j.Logger; +import org.w3c.dom.Element; + +/** + * The unit database of a voice + * + * @author Marc Schröder + * + */ +public class UnitDatabase { + protected TargetCostFunction targetCostFunction; + protected JoinCostFunction joinCostFunction; + protected StatisticalCostFunction sCostFunction = null; + protected UnitFileReader unitReader; + protected int numUnits; + protected CART preselectionCART; + protected TimelineReader audioTimeline; + protected TimelineReader basenameTimeline; + protected int backtrace; + protected Logger logger = MaryUtils.getLogger("UnitDatabase"); + + public UnitDatabase() { + } + + public void load(TargetCostFunction aTargetCostFunction, JoinCostFunction aJoinCostFunction, UnitFileReader aUnitReader, + CART aPreselectionCART, TimelineReader anAudioTimeline, TimelineReader aBasenameTimeline, int backtraceLeafSize) { + this.targetCostFunction = aTargetCostFunction; + this.joinCostFunction = aJoinCostFunction; + this.unitReader = aUnitReader; + this.numUnits = (unitReader != null ? unitReader.getNumberOfUnits() : 0); + this.preselectionCART = aPreselectionCART; + this.audioTimeline = anAudioTimeline; + this.basenameTimeline = aBasenameTimeline; + this.backtrace = backtraceLeafSize; + } + + public void load(TargetCostFunction aTargetCostFunction, JoinCostFunction aJoinCostFunction, + StatisticalCostFunction asCostFunction, UnitFileReader aUnitReader, CART aPreselectionCART, + TimelineReader anAudioTimeline, TimelineReader aBasenameTimeline, int backtraceLeafSize) { + this.targetCostFunction = aTargetCostFunction; + this.joinCostFunction = aJoinCostFunction; + this.sCostFunction = asCostFunction; + this.unitReader = aUnitReader; + this.numUnits = (unitReader != null ? unitReader.getNumberOfUnits() : 0); + this.preselectionCART = aPreselectionCART; + this.audioTimeline = anAudioTimeline; + this.basenameTimeline = aBasenameTimeline; + this.backtrace = backtraceLeafSize; + } + + public TargetCostFunction getTargetCostFunction() { + return targetCostFunction; + } + + public JoinCostFunction getJoinCostFunction() { + return joinCostFunction; + } + + public UnitFileReader getUnitFileReader() { + return unitReader; + } + + public TimelineReader getAudioTimeline() { + return audioTimeline; + } + + public StatisticalCostFunction getSCostFunction() { + return sCostFunction; + } + + /** + * Preselect a set of candidates that could be used to realise the given target. + * + * @param target + * a Target object representing an optimal unit + * @return an unsorted ArrayList of ViterbiCandidates, each containing the (same) target and a + * (different) Unit object + */ + public List getCandidates(Target target) { + // BEGIN blacklisting + // The point of this is to get the value of the "blacklist" attribute in the first child element of the MaryXML + // and store it in the blacklist String variable. + // This code seems rather inelegant; perhaps there is a better way to access the MaryXML from this method? + String blacklist = ""; + String unitBasename = "This must never be null or the empty string!"; // otherwise candidate selection fails! + Element targetElement = target.getMaryxmlElement(); + blacklist = DomUtils.getAttributeFromClosestAncestorOfAnyKind(targetElement, "blacklist"); + // END blacklisting + + // logger.debug("Looking for candidates in cart "+target.getName()); + // get the cart tree and extract the candidates + int[] clist = (int[]) preselectionCART.interpret(target, backtrace); + logger.debug("For target " + target + ", selected " + clist.length + " units"); + + // Now, clist is an array of unit indexes. + List candidates = new ArrayList(); + for (int i = 0; i < clist.length; i++) { + // The target is the same for all these candidates in the queue + // remember the actual unit: + Unit unit = unitReader.getUnit(clist[i]); + candidates.add(new ViterbiCandidate(target, unit, targetCostFunction)); + } + + // Blacklisting without crazy performance drop: + // just remove candidates again if their basenames are blacklisted + java.util.Iterator candIt = candidates.iterator(); + while (candIt.hasNext()) { + ViterbiCandidate candidate = candIt.next(); + unitBasename = getFilename(candidate.getUnit()); + if (blacklist.contains(unitBasename)) { + candIt.remove(); + } + } + + return candidates; + } + + /** + * For debugging, return the basename of the original audio file from which the unit is coming, as well as the start time in + * that file. + * + * @param unit + * @return a String containing basename followed by a space and the unit's start time, in seconds, from the beginning of the + * file. If no basenameTimeline was specified for this voice, returns the string "unknown origin". + */ + public String getFilenameAndTime(Unit unit) { + if (basenameTimeline == null) + return "unknown origin"; + long[] offset = new long[1]; + try { + Datagram[] datagrams = basenameTimeline.getDatagrams(unit.startTime, 1, unitReader.getSampleRate(), offset); + Datagram filenameData = datagrams[0]; + float time = (float) offset[0] / basenameTimeline.getSampleRate(); + String filename = new String(filenameData.getData(), "UTF-8"); + return filename + " " + time; + } catch (Exception e) { + logger.warn("Problem getting filename and time for unit " + unit.index + " at time " + unit.startTime, e); + return "unknown origin"; + } + } + + /** + * For debugging, return the basename of the original audio file from which the unit is coming. + * + * @param unit + * @return a String containing basename. If no basenameTimeline was specified for this voice, returns the string + * "unknown origin". + */ + public String getFilename(Unit unit) { + // if (basenameTimeline == null) return "unknown origin"; + try { + Datagram filenameData = basenameTimeline.getDatagram(unit.startTime); + String filename = new String(filenameData.getData(), "UTF-8"); + return filename; + } catch (Exception e) { + logger.warn("Problem getting filename for unit " + unit.index, e); + return "unknown origin"; + } + } + +} diff --git a/marytts-unitselection/src/main/java/marytts/unitselection/data/UnitFileReader.java b/marytts-unitselection/src/main/java/marytts/unitselection/data/UnitFileReader.java new file mode 100644 index 0000000000..4f8a9b695e --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/unitselection/data/UnitFileReader.java @@ -0,0 +1,200 @@ +/** + * Portions Copyright 2006 DFKI GmbH. + * Portions Copyright 2001 Sun Microsystems, Inc. + * Portions Copyright 1999-2001 Language Technologies Institute, + * Carnegie Mellon University. + * All Rights Reserved. Use is subject to license terms. + * + * Permission is hereby granted, free of charge, to use and distribute + * this software and its documentation without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of this work, and to + * permit persons to whom this work is furnished to do so, subject to + * the following conditions: + * + * 1. The code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * 2. Any modifications must be clearly marked as such. + * 3. Original authors' names are not deleted. + * 4. The authors' names are not used to endorse or promote products + * derived from this software without specific prior written + * permission. + * + * DFKI GMBH AND THE CONTRIBUTORS TO THIS WORK DISCLAIM ALL WARRANTIES WITH + * REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL DFKI GMBH NOR THE + * CONTRIBUTORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL + * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR + * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS + * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF + * THIS SOFTWARE. + */ +package marytts.unitselection.data; + +import java.io.BufferedInputStream; +import java.io.DataInputStream; +import java.io.FileInputStream; +import java.io.IOException; + +import marytts.exceptions.MaryConfigurationException; +import marytts.util.data.MaryHeader; + +/** + * Loads a unit file in memory and provides accessors to the start times and durations. + * + * @author sacha + * + */ +public class UnitFileReader { + + private MaryHeader hdr = null; + private int numberOfUnits = 0; + private int sampleRate = 0; + Unit[] units; // this has visibility "default" rather than private so that other classes in the same package can access it + // directly, for efficiency reasons + + /****************/ + /* CONSTRUCTORS */ + /****************/ + + /** + * Empty constructor; need to call load() separately. + * + * @see #load(String) + */ + public UnitFileReader() { + } + + /** + * Create a unit file reader from the given unit file + * + * @param fileName + * the unit file to read + * @throws IOException + * if a problem occurs while reading + */ + public UnitFileReader(String fileName) throws IOException, MaryConfigurationException { + load(fileName); + } + + /** + * Load the given unit file + * + * @param fileName + * the unit file to read + * @throws IOException + * if a problem occurs while reading + */ + public void load(String fileName) throws IOException, MaryConfigurationException { + /* Open the file */ + DataInputStream dis = new DataInputStream(new BufferedInputStream(new FileInputStream(fileName))); + /* Load the Mary header */ + hdr = new MaryHeader(dis); + if (hdr.getType() != MaryHeader.UNITS) { + throw new MaryConfigurationException("File [" + fileName + "] is not a valid Mary Units file."); + } + /* Read the number of units */ + numberOfUnits = dis.readInt(); + if (numberOfUnits < 0) { + throw new MaryConfigurationException("File [" + fileName + "] has a negative number of units. Aborting."); + } + /* Read the sample rate */ + sampleRate = dis.readInt(); + if (sampleRate < 0) { + throw new MaryConfigurationException("File [" + fileName + "] has a negative number sample rate. Aborting."); + } + units = new Unit[numberOfUnits]; + /* Read the start times and durations */ + for (int i = 0; i < numberOfUnits; i++) { + long startTime = dis.readLong(); + int duration = dis.readInt(); + units[i] = new Unit(startTime, duration, i); + } + } + + /*****************/ + /* OTHER METHODS */ + /*****************/ + + /** + * Get the number of units in the file. + * + * @return The number of units. + */ + public int getNumberOfUnits() { + return (numberOfUnits); + } + + /** + * Get the sample rate of the file. + * + * @return The sample rate, in Hz. + */ + public int getSampleRate() { + return (sampleRate); + } + + /** + * Return the unit number i. + * + * @param i + * The index of the considered unit. + * @return The considered unit. + */ + public Unit getUnit(int i) { + return units[i]; + } + + /** + * Return an array of units from their indexes. + * + * @param i + * The indexes of the considered units. + * @return The array of considered units. + */ + public Unit[] getUnit(int[] i) { + Unit[] ret = new Unit[i.length]; + for (int k = 0; k < i.length; k++) { + ret[k] = getUnit(i[k]); + } + return (ret); + } + + /** + * Return the unit following the given unit in the original database. + * + * @param u + * a unit + * @return the next unit in the database, or null if there is no such unit. + */ + public Unit getNextUnit(Unit u) { + if (u == null || u.index >= units.length - 1 || u.index < 0) + return null; + return units[u.index + 1]; + } + + /** + * Return the unit preceding the given unit in the original database. + * + * @param u + * a unit + * @return the previous unit in the database, or null if there is no such unit. + */ + public Unit getPreviousUnit(Unit u) { + if (u == null || u.index >= units.length || u.index <= 0) + return null; + return units[u.index - 1]; + } + + /** + * Determine whether the unit number i is an "edge" unit, i.e. a unit marking the start or the end of an utterance. + * + * @param i + * The index of the considered unit. + * @return true if the unit is an edge unit in the unit file, false otherwise + */ + public boolean isEdgeUnit(int i) { + return units[i].isEdgeUnit(); + } + +} diff --git a/marytts-unitselection/src/main/java/marytts/unitselection/interpolation/InterpolatingSynthesizer.java b/marytts-unitselection/src/main/java/marytts/unitselection/interpolation/InterpolatingSynthesizer.java new file mode 100644 index 0000000000..0d035f4342 --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/unitselection/interpolation/InterpolatingSynthesizer.java @@ -0,0 +1,203 @@ +/** + * Copyright 2007 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.unitselection.interpolation; + +import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.Iterator; +import java.util.List; + +import javax.sound.sampled.AudioInputStream; + +import marytts.datatypes.MaryXML; +import marytts.exceptions.SynthesisException; +import marytts.modules.synthesis.Voice; +import marytts.modules.synthesis.WaveformSynthesizer; +import marytts.signalproc.process.FramewiseMerger; +import marytts.signalproc.process.LSFInterpolator; +import marytts.unitselection.UnitSelectionVoice; +import marytts.unitselection.concat.BaseUnitConcatenator; +import marytts.unitselection.concat.UnitConcatenator; +import marytts.unitselection.select.SelectedUnit; +import marytts.unitselection.select.UnitSelector; +import marytts.util.MaryUtils; +import marytts.util.data.BufferedDoubleDataSource; +import marytts.util.data.Datagram; +import marytts.util.data.DoubleDataSource; +import marytts.util.data.audio.AudioDoubleDataSource; +import marytts.util.data.audio.DDSAudioInputStream; +import marytts.util.dom.MaryDomUtils; + +import org.apache.log4j.Logger; +import org.w3c.dom.Element; + +/** + * @author marc + * + */ +public class InterpolatingSynthesizer implements WaveformSynthesizer { + protected Logger logger; + + /** + * + */ + public InterpolatingSynthesizer() { + } + + /** + * Start up the waveform synthesizer. This must be called once before calling synthesize(). + */ + public void startup() throws Exception { + logger = MaryUtils.getLogger("InterpolatingSynthesizer"); + // Register interpolating voice: + Voice.registerVoice(new InterpolatingVoice(this, "interpolatingvoice")); + logger.info("started."); + } + + /** + * Perform a power-on self test by processing some example input data. + * + * @throws Error + * if the module does not work properly. + */ + public void powerOnSelfTest() throws Error { + } + + /** + * {@inheritDoc} + */ + public AudioInputStream synthesize(List tokensAndBoundaries, Voice voice, String outputParams) + throws SynthesisException { + if (tokensAndBoundaries.size() == 0) + return null; + + // 1. determine the two voices involved; + Element first = (Element) tokensAndBoundaries.get(0); + Element voiceElement = (Element) MaryDomUtils.getAncestor(first, MaryXML.VOICE); + String name = voiceElement.getAttribute("name"); + // name has the form: voice1 with XY% voice2 + // We trust that InerpolatingVoice.hasName() has done its job to verify that + // name actually has that form. + String[] parts = name.split("\\s+"); + assert parts.length == 4; + assert parts[1].equals("with"); + assert parts[2].endsWith("%"); + int percent = Integer.parseInt(parts[2].substring(0, parts[2].length() - 1)); + Voice voice1 = Voice.getVoice(parts[0]); + assert voice1 != null; + Voice voice2 = Voice.getVoice(parts[3]); + assert voice2 != null; + + // 2. do unit selection with each; + if (!(voice1 instanceof UnitSelectionVoice)) { + throw new IllegalArgumentException("Voices of type " + voice.getClass().getName() + " not supported!"); + } + if (!(voice2 instanceof UnitSelectionVoice)) { + throw new IllegalArgumentException("Voices of type " + voice.getClass().getName() + " not supported!"); + } + UnitSelectionVoice usv1 = (UnitSelectionVoice) voice1; + UnitSelectionVoice usv2 = (UnitSelectionVoice) voice2; + + UnitSelector unitSel1 = usv1.getUnitSelector(); + List selectedUnits1 = unitSel1.selectUnits(tokensAndBoundaries, voice); + UnitSelector unitSel2 = usv2.getUnitSelector(); + List selectedUnits2 = unitSel2.selectUnits(tokensAndBoundaries, voice); + assert selectedUnits1.size() == selectedUnits2.size() : "Unexpected difference in number of units: " + + selectedUnits1.size() + " vs. " + selectedUnits2.size(); + int numUnits = selectedUnits1.size(); + + // 3. do unit concatenation with each, retrieve actual unit durations from list of units; + UnitConcatenator unitConcatenator1 = usv1.getConcatenator(); + AudioInputStream audio1; + try { + audio1 = unitConcatenator1.getAudio(selectedUnits1); + } catch (IOException ioe) { + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw); + for (Iterator selIt = selectedUnits1.iterator(); selIt.hasNext();) + pw.println(selIt.next()); + throw new SynthesisException("For voice " + voice1.getName() + ", problems generating audio for unit chain: " + + sw.toString(), ioe); + } + DoubleDataSource audioSource1 = new AudioDoubleDataSource(audio1); + UnitConcatenator unitConcatenator2 = usv2.getConcatenator(); + AudioInputStream audio2; + try { + audio2 = unitConcatenator2.getAudio(selectedUnits2); + } catch (IOException ioe) { + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw); + for (Iterator selIt = selectedUnits2.iterator(); selIt.hasNext();) + pw.println(selIt.next()); + throw new SynthesisException("For voice " + voice2.getName() + ", problems generating audio for unit chain: " + + sw.toString(), ioe); + } + DoubleDataSource audioSource2 = new AudioDoubleDataSource(audio2); + // Retrieve actual durations from list of units: + int sampleRate1 = (int) usv1.dbAudioFormat().getSampleRate(); + double[] label1 = new double[numUnits]; + double t1 = 0; + int sampleRate2 = (int) usv2.dbAudioFormat().getSampleRate(); + double[] label2 = new double[numUnits]; + double t2 = 0; + for (int i = 0; i < numUnits; i++) { + SelectedUnit u1 = selectedUnits1.get(i); + SelectedUnit u2 = selectedUnits2.get(i); + BaseUnitConcatenator.UnitData ud1 = (BaseUnitConcatenator.UnitData) u1.getConcatenationData(); + int unitDuration1 = ud1.getUnitDuration(); + if (unitDuration1 < 0) { // was not set by the unit concatenator, have to count ourselves + unitDuration1 = 0; + Datagram[] d = ud1.getFrames(); + for (int id = 0; id < d.length; id++) { + unitDuration1 += d[id].getDuration(); + } + } + t1 += unitDuration1 / (float) sampleRate1; + label1[i] = t1; + BaseUnitConcatenator.UnitData ud2 = (BaseUnitConcatenator.UnitData) u2.getConcatenationData(); + int unitDuration2 = ud2.getUnitDuration(); + if (unitDuration2 < 0) { // was not set by the unit concatenator, have to count ourselves + unitDuration2 = 0; + Datagram[] d = ud2.getFrames(); + for (int id = 0; id < d.length; id++) { + unitDuration2 += d[id].getDuration(); + } + } + t2 += unitDuration2 / (float) sampleRate2; + label2[i] = t2; + logger.debug(usv1.getName() + " [" + u1.getTarget() + "] " + label1[i] + " -- " + usv2.getName() + " [" + + u2.getTarget() + "] " + label2[i]); + } + + // 4. with these unit durations, run the LSFInterpolator on the two audio streams. + int frameLength = Integer.getInteger("signalproc.lpcanalysisresynthesis.framelength", 512).intValue(); + int predictionOrder = Integer.getInteger("signalproc.lpcanalysisresynthesis.predictionorder", 20).intValue(); + double r = (double) percent / 100; + assert r >= 0; + assert r <= 1; + FramewiseMerger foas = new FramewiseMerger(audioSource1, frameLength, sampleRate1, new BufferedDoubleDataSource(label1), + audioSource2, sampleRate2, new BufferedDoubleDataSource(label2), new LSFInterpolator(predictionOrder, r)); + DDSAudioInputStream outputAudio = new DDSAudioInputStream(new BufferedDoubleDataSource(foas), audio1.getFormat()); + + return outputAudio; + } + +} diff --git a/marytts-unitselection/src/main/java/marytts/unitselection/interpolation/InterpolatingVoice.java b/marytts-unitselection/src/main/java/marytts/unitselection/interpolation/InterpolatingVoice.java new file mode 100644 index 0000000000..903b5c4a29 --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/unitselection/interpolation/InterpolatingVoice.java @@ -0,0 +1,119 @@ +/** + * Copyright 2007 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.unitselection.interpolation; + +import java.util.Locale; + +import javax.sound.sampled.AudioFormat; + +import marytts.exceptions.MaryConfigurationException; +import marytts.modules.phonemiser.Allophone; +import marytts.modules.phonemiser.AllophoneSet; +import marytts.modules.synthesis.Voice; + +/** + * @author marc + * + */ +public class InterpolatingVoice extends Voice { + + public static boolean isInterpolatingVoiceName(String name) { + if (name == null) + return false; + String[] parts = name.split("\\s+"); + if (parts.length != 4) + return false; + if (!parts[1].equals("with")) + return false; + if (!parts[2].endsWith("%")) + return false; + int percent; + try { + percent = Integer.parseInt(parts[2].substring(0, parts[2].length() - 1)); + } catch (NumberFormatException nfe) { + return false; + } + if (Voice.getVoice(parts[0]) == null) + return false; + if (Voice.getVoice(parts[3]) == null) + return false; + return true; + } + + protected Voice firstVoice = null; + + public InterpolatingVoice(InterpolatingSynthesizer is, String name) throws MaryConfigurationException { + super(name, null, null, is, null); + if (isInterpolatingVoiceName(name)) { + String[] parts = name.split("\\s+"); + firstVoice = Voice.getVoice(parts[0]); + } + } + + /** + * Determine whether this voice has the given name. For the InterpolatingVoice, the meaning of the name is different from a + * "normal" voice. It is a specification of how to interpolate two voices. The syntax is:
+ * voice1 with XY% voice2
+ *
+ * where voice1 and voice2 must be existing voices, and XY is an integer between 0 and 100. + * + * @return true if name matches the specification, false otherwise + */ + /* + * public boolean hasName(String name) { if (name == null) return false; String[] parts = name.split("\\s+"); if (parts.length + * != 4) return false; if (!parts[1].equals("with")) return false; if (!parts[2].endsWith("%")) return false; int percent; try + * { percent = Integer.parseInt(parts[2].substring(0, parts[2].length()-1)); } catch (NumberFormatException nfe) { return + * false; } if (Voice.getVoice(parts[0]) == null) return false; if (Voice.getVoice(parts[3]) == null) return false; return + * true; } + */ + + // Forward most of the public methods which are meaningful in a unit selection context to firstVoice: + + public AllophoneSet getAllophoneSet() { + if (firstVoice == null) + return null; + return firstVoice.getAllophoneSet(); + } + + public Allophone getAllophone(String phoneSymbol) { + if (firstVoice == null) + return null; + return firstVoice.getAllophone(phoneSymbol); + } + + public Locale getLocale() { + if (firstVoice == null) + return null; + return firstVoice.getLocale(); + } + + public AudioFormat dbAudioFormat() { + if (firstVoice == null) + return null; + return firstVoice.dbAudioFormat(); + } + + public Gender gender() { + if (firstVoice == null) + return null; + return firstVoice.gender(); + } + +} diff --git a/marytts-unitselection/src/main/java/marytts/unitselection/select/DiphoneFFRTargetCostFunction.java b/marytts-unitselection/src/main/java/marytts/unitselection/select/DiphoneFFRTargetCostFunction.java new file mode 100644 index 0000000000..19bda4328c --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/unitselection/select/DiphoneFFRTargetCostFunction.java @@ -0,0 +1,145 @@ +/** + * Copyright 2006 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.unitselection.select; + +import java.io.IOException; +import java.io.InputStream; + +import marytts.exceptions.MaryConfigurationException; +import marytts.features.FeatureDefinition; +import marytts.features.FeatureProcessorManager; +import marytts.features.FeatureVector; +import marytts.features.Target; +import marytts.features.DiphoneTarget; +import marytts.features.HalfPhoneTarget; +import marytts.unitselection.data.DiphoneUnit; +import marytts.unitselection.data.FeatureFileReader; +import marytts.unitselection.data.HalfPhoneFeatureFileReader; +import marytts.unitselection.data.Unit; + +public class DiphoneFFRTargetCostFunction implements TargetCostFunction { + protected FFRTargetCostFunction tcfForHalfphones; + + public DiphoneFFRTargetCostFunction() { + } + + /** + * Initialise the data needed to do a target cost computation. + * + * @param featureFileName + * name of a file containing the unit features + * @param weightsFile + * an optional weights file -- if non-null, contains feature weights that override the ones present in the feature + * file. + * @param featProc + * a feature processor manager which can provide feature processors to compute the features for a target at run + * time + * @throws IOException + */ + @Override + public void load(String featureFileName, InputStream weightsStream, FeatureProcessorManager featProc) throws IOException, + MaryConfigurationException { + FeatureFileReader ffr = FeatureFileReader.getFeatureFileReader(featureFileName); + load(ffr, weightsStream, featProc); + } + + @Override + public void load(FeatureFileReader ffr, InputStream weightsStream, FeatureProcessorManager featProc) throws IOException { + if (ffr instanceof HalfPhoneFeatureFileReader) { + tcfForHalfphones = new HalfPhoneFFRTargetCostFunction(); + } else { + tcfForHalfphones = new FFRTargetCostFunction(); + } + tcfForHalfphones.load(ffr, weightsStream, featProc); + } + + /** + * Provide access to the Feature Definition used. + * + * @return the feature definition object. + */ + public FeatureDefinition getFeatureDefinition() { + return tcfForHalfphones.getFeatureDefinition(); + } + + /** + * Get the string representation of the feature value associated with the given unit + * + * @param unit + * the unit whose feature value is requested + * @param featureName + * name of the feature requested + * @return a string representation of the feature value + * @throws IllegalArgumentException + * if featureName is not a known feature + */ + public String getFeature(Unit unit, String featureName) { + return tcfForHalfphones.getFeature(unit, featureName); + } + + public FeatureVector getFeatureVector(Unit unit) { + return tcfForHalfphones.featureVectors[unit.index]; + } + + /** + * Compute the goodness-of-fit of a given unit for a given target. + * + * @param target + * @param unit + * @return a non-negative number; smaller values mean better fit, i.e. smaller cost. + */ + public double cost(Target target, Unit unit) { + if (target instanceof HalfPhoneTarget) + return tcfForHalfphones.cost(target, unit); + if (!(target instanceof DiphoneTarget)) + throw new IllegalArgumentException("This target cost function can only be called for diphone and half-phone targets!"); + if (!(unit instanceof DiphoneUnit)) + throw new IllegalArgumentException("Diphone targets need diphone units!"); + DiphoneTarget dt = (DiphoneTarget) target; + DiphoneUnit du = (DiphoneUnit) unit; + return tcfForHalfphones.cost(dt.left, du.left) + tcfForHalfphones.cost(dt.right, du.right); + } + + /** + * Compute the features for a given target, and store them in the target. + * + * @param target + * the target for which to compute the features + * @see Target#getFeatureVector() + */ + public void computeTargetFeatures(Target target) { + if (!(target instanceof DiphoneTarget)) { + tcfForHalfphones.computeTargetFeatures(target); + } else { + DiphoneTarget dt = (DiphoneTarget) target; + tcfForHalfphones.computeTargetFeatures(dt.left); + tcfForHalfphones.computeTargetFeatures(dt.right); + + } + } + + public FeatureVector[] getFeatureVectors() { + if (tcfForHalfphones != null) { + return tcfForHalfphones.getFeatureVectors(); + } + return null; + } + +} diff --git a/marytts-unitselection/src/main/java/marytts/unitselection/select/DiphoneUnitSelector.java b/marytts-unitselection/src/main/java/marytts/unitselection/select/DiphoneUnitSelector.java new file mode 100644 index 0000000000..340a3970fb --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/unitselection/select/DiphoneUnitSelector.java @@ -0,0 +1,72 @@ +/** + * Copyright 2006 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.unitselection.select; + +import java.util.ArrayList; +import java.util.List; + +import marytts.unitselection.data.UnitDatabase; +import marytts.features.Target; +import marytts.features.DiphoneTarget; +import marytts.features.HalfPhoneTarget; + +import org.w3c.dom.Element; + +public class DiphoneUnitSelector extends UnitSelector { + + /** + * Initialise the unit selector. Need to call load() separately. + * + * @see #load(UnitDatabase) + */ + public DiphoneUnitSelector() throws Exception { + super(); + } + + /** + * Create the list of targets from the XML elements to synthesize. + * + * @param segmentsAndBoundaries + * a list of MaryXML phone and boundary elements + * @return a list of Target objects + */ + protected List createTargets(List segmentsAndBoundaries) { + List targets = new ArrayList(); + // TODO: how can we know the silence symbol here? + String silenceSymbol = "_"; // in sampa + + // Insert an initial silence with duration 0 (needed as context) + HalfPhoneTarget prev = new HalfPhoneTarget(silenceSymbol + "_R", null, false); + for (Element sOrB : segmentsAndBoundaries) { + String phone = getPhoneSymbol(sOrB); + HalfPhoneTarget leftHalfPhone = new HalfPhoneTarget(phone + "_L", sOrB, true); // left half + HalfPhoneTarget rightHalfPhone = new HalfPhoneTarget(phone + "_R", sOrB, false); // right half + targets.add(new DiphoneTarget(prev, leftHalfPhone)); + prev = rightHalfPhone; + } + // Make sure there is a final silence + if (!prev.isSilence()) { + HalfPhoneTarget silence = new HalfPhoneTarget(silenceSymbol + "_L", null, true); + targets.add(new DiphoneTarget(prev, silence)); + } + return targets; + } + +} diff --git a/marytts-unitselection/src/main/java/marytts/unitselection/select/FFRTargetCostFunction.java b/marytts-unitselection/src/main/java/marytts/unitselection/select/FFRTargetCostFunction.java new file mode 100644 index 0000000000..c20b6d265a --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/unitselection/select/FFRTargetCostFunction.java @@ -0,0 +1,387 @@ +/** + * Copyright 2006 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.unitselection.select; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; + +import marytts.exceptions.MaryConfigurationException; +import marytts.features.FeatureDefinition; +import marytts.features.FeatureProcessorManager; +import marytts.features.FeatureVector; +import marytts.features.TargetFeatureComputer; +import marytts.features.Target; +import marytts.server.MaryProperties; +import marytts.signalproc.display.Histogram; +import marytts.unitselection.data.FeatureFileReader; +import marytts.unitselection.data.Unit; +import marytts.unitselection.weightingfunctions.WeightFunc; +import marytts.unitselection.weightingfunctions.WeightFunctionManager; +import marytts.util.MaryUtils; + +public class FFRTargetCostFunction implements TargetCostFunction { + protected WeightFunc[] weightFunction; + protected TargetFeatureComputer targetFeatureComputer; + protected FeatureVector[] featureVectors; + protected FeatureDefinition featureDefinition; + protected boolean[] weightsNonZero; + + protected boolean debugShowCostGraph = false; + protected double[] cumulWeightedCosts = null; + protected int nCostComputations = 0; + + public FFRTargetCostFunction() { + } + + /** + * Compute the goodness-of-fit of a given unit for a given target. + * + * @param target + * @param unit + * @return a non-negative number; smaller values mean better fit, i.e. smaller cost. + */ + public double cost(Target target, Unit unit) { + return cost(target, unit, featureDefinition, weightFunction); + } + + protected double cost(Target target, Unit unit, FeatureDefinition weights, WeightFunc[] weightFunctions) { + nCostComputations++; // for debug + FeatureVector targetFeatures = target.getFeatureVector(); + assert targetFeatures != null : "Target " + target + " does not have pre-computed feature vector"; + FeatureVector unitFeatures = featureVectors[unit.index]; + int nBytes = targetFeatures.byteValuedDiscreteFeatures.length; + int nShorts = targetFeatures.shortValuedDiscreteFeatures.length; + int nFloats = targetFeatures.continuousFeatures.length; + assert nBytes == unitFeatures.byteValuedDiscreteFeatures.length; + assert nShorts == unitFeatures.shortValuedDiscreteFeatures.length; + assert nFloats == unitFeatures.continuousFeatures.length; + + float[] weightVector = weights.getFeatureWeights(); + // Now the actual computation + double cost = 0; + // byte-valued features: + if (nBytes > 0) { + for (int i = 0; i < nBytes; i++) { + if (weightsNonZero[i]) { + float weight = weightVector[i]; + if (featureDefinition.hasSimilarityMatrix(i)) { + byte targetFeatValueIndex = targetFeatures.byteValuedDiscreteFeatures[i]; + byte unitFeatValueIndex = unitFeatures.byteValuedDiscreteFeatures[i]; + float similarity = featureDefinition.getSimilarity(i, unitFeatValueIndex, targetFeatValueIndex); + cost += similarity * weight; + if (debugShowCostGraph) + cumulWeightedCosts[i] += similarity * weight; + } else if (targetFeatures.byteValuedDiscreteFeatures[i] != unitFeatures.byteValuedDiscreteFeatures[i]) { + cost += weight; + if (debugShowCostGraph) + cumulWeightedCosts[i] += weight; + } + } + } + } + // short-valued features: + if (nShorts > 0) { + for (int i = nBytes, n = nBytes + nShorts; i < n; i++) { + if (weightsNonZero[i]) { + float weight = weightVector[i]; + // if (targetFeatures.getShortFeature(i) != unitFeatures.getShortFeature(i)) { + if (targetFeatures.shortValuedDiscreteFeatures[i - nBytes] != unitFeatures.shortValuedDiscreteFeatures[i + - nBytes]) { + cost += weight; + if (debugShowCostGraph) + cumulWeightedCosts[i] += weight; + } + } + } + } + // continuous features: + if (nFloats > 0) { + int nDiscrete = nBytes + nShorts; + for (int i = nDiscrete, n = nDiscrete + nFloats; i < n; i++) { + if (weightsNonZero[i]) { + float weight = weightVector[i]; + // float a = targetFeatures.getContinuousFeature(i); + float a = targetFeatures.continuousFeatures[i - nDiscrete]; + // float b = unitFeatures.getContinuousFeature(i); + float b = unitFeatures.continuousFeatures[i - nDiscrete]; + // if (!Float.isNaN(a) && !Float.isNaN(b)) { + // Implementation of isNaN() is: (v != v). + if (!(a != a) && !(b != b)) { + double myCost = weightFunctions[i - nDiscrete].cost(a, b); + cost += weight * myCost; + if (debugShowCostGraph) { + cumulWeightedCosts[i] += weight * myCost; + } + } // and if it is NaN, simply compute no cost + } + } + } + return cost; + } + + /** + * Compute the goodness-of-fit between given unit and given target for a given feature + * + * @param target + * target unit + * @param unit + * candidate unit + * @param featureName + * feature name + * @return a non-negative number; smaller values mean better fit, i.e. smaller cost. + * @throws IllegalArgumentException + * if featureName not available in featureDefinition + */ + public double featureCost(Target target, Unit unit, String featureName) { + return featureCost(target, unit, featureName, featureDefinition, weightFunction); + } + + protected double featureCost(Target target, Unit unit, String featureName, FeatureDefinition weights, + WeightFunc[] weightFunctions) { + if (!this.featureDefinition.hasFeature(featureName)) { + throw new IllegalArgumentException("this feature does not exists in feature definition"); + } + + FeatureVector targetFeatures = target.getFeatureVector(); + assert targetFeatures != null : "Target " + target + " does not have pre-computed feature vector"; + FeatureVector unitFeatures = featureVectors[unit.index]; + int nBytes = targetFeatures.byteValuedDiscreteFeatures.length; + int nShorts = targetFeatures.shortValuedDiscreteFeatures.length; + int nFloats = targetFeatures.continuousFeatures.length; + assert nBytes == unitFeatures.byteValuedDiscreteFeatures.length; + assert nShorts == unitFeatures.shortValuedDiscreteFeatures.length; + assert nFloats == unitFeatures.continuousFeatures.length; + + int featureIndex = this.featureDefinition.getFeatureIndex(featureName); + float[] weightVector = weights.getFeatureWeights(); + double cost = 0; + + if (featureIndex < nBytes) { + if (weightsNonZero[featureIndex]) { + float weight = weightVector[featureIndex]; + if (featureDefinition.hasSimilarityMatrix(featureIndex)) { + byte targetFeatValueIndex = targetFeatures.byteValuedDiscreteFeatures[featureIndex]; + byte unitFeatValueIndex = unitFeatures.byteValuedDiscreteFeatures[featureIndex]; + float similarity = featureDefinition.getSimilarity(featureIndex, unitFeatValueIndex, targetFeatValueIndex); + cost = similarity * weight; + if (debugShowCostGraph) + cumulWeightedCosts[featureIndex] += similarity * weight; + } else if (targetFeatures.byteValuedDiscreteFeatures[featureIndex] != unitFeatures.byteValuedDiscreteFeatures[featureIndex]) { + cost = weight; + if (debugShowCostGraph) + cumulWeightedCosts[featureIndex] += weight; + } + } + } else if (featureIndex < nShorts + nBytes) { + if (weightsNonZero[featureIndex]) { + float weight = weightVector[featureIndex]; + // if (targetFeatures.getShortFeature(i) != unitFeatures.getShortFeature(i)) { + if (targetFeatures.shortValuedDiscreteFeatures[featureIndex - nBytes] != unitFeatures.shortValuedDiscreteFeatures[featureIndex + - nBytes]) { + cost = weight; + if (debugShowCostGraph) + cumulWeightedCosts[featureIndex] += weight; + } + } + } else { + int nDiscrete = nBytes + nShorts; + if (weightsNonZero[featureIndex]) { + float weight = weightVector[featureIndex]; + // float a = targetFeatures.getContinuousFeature(i); + float a = targetFeatures.continuousFeatures[featureIndex - nDiscrete]; + // float b = unitFeatures.getContinuousFeature(i); + float b = unitFeatures.continuousFeatures[featureIndex - nDiscrete]; + // if (!Float.isNaN(a) && !Float.isNaN(b)) { + // Implementation of isNaN() is: (v != v). + if (!(a != a) && !(b != b)) { + double myCost = weightFunctions[featureIndex - nDiscrete].cost(a, b); + cost = weight * myCost; + if (debugShowCostGraph) { + cumulWeightedCosts[featureIndex] += weight * myCost; + } + } // and if it is NaN, simply compute no cost + } + } + return cost; + } + + /** + * Initialise the data needed to do a target cost computation. + * + * @param featureFileName + * name of a file containing the unit features + * @param weightsFile + * an optional weights file -- if non-null, contains feature weights that override the ones present in the feature + * file. + * @param featProc + * a feature processor manager which can provide feature processors to compute the features for a target at run + * time + * @throws IOException + * @throws MaryConfigurationException + */ + @Override + public void load(String featureFileName, InputStream weightsStream, FeatureProcessorManager featProc) throws IOException, + MaryConfigurationException { + FeatureFileReader ffr = FeatureFileReader.getFeatureFileReader(featureFileName); + load(ffr, weightsStream, featProc); + } + + @Override + public void load(FeatureFileReader ffr, InputStream weightsStream, FeatureProcessorManager featProc) throws IOException { + this.featureDefinition = ffr.getFeatureDefinition(); + this.featureVectors = ffr.getFeatureVectors(); + if (weightsStream != null) { + MaryUtils.getLogger("TargetCostFeatures").debug("Overwriting target cost weights from file"); + // overwrite weights from file + + FeatureDefinition newWeights = new FeatureDefinition( + new BufferedReader(new InputStreamReader(weightsStream, "UTF-8")), true); + if (!newWeights.featureEquals(featureDefinition)) { + throw new IOException("Weights file: feature definition incompatible with feature file"); + } + featureDefinition = newWeights; + } + weightFunction = new WeightFunc[featureDefinition.getNumberOfContinuousFeatures()]; + WeightFunctionManager wfm = new WeightFunctionManager(); + int nDiscreteFeatures = featureDefinition.getNumberOfByteFeatures() + featureDefinition.getNumberOfShortFeatures(); + for (int i = 0; i < weightFunction.length; i++) { + String weightFunctionName = featureDefinition.getWeightFunctionName(nDiscreteFeatures + i); + if ("".equals(weightFunctionName)) + weightFunction[i] = wfm.getWeightFunction("linear"); + else + weightFunction[i] = wfm.getWeightFunction(weightFunctionName); + } + // TODO: If the target feature computer had direct access to the feature definition, it could do some consistency checking + this.targetFeatureComputer = new TargetFeatureComputer(featProc, featureDefinition.getFeatureNames()); + + rememberWhichWeightsAreNonZero(); + + if (MaryProperties.getBoolean("debug.show.cost.graph")) { + debugShowCostGraph = true; + cumulWeightedCosts = new double[featureDefinition.getNumberOfFeatures()]; + TargetCostReporter tcr2 = new TargetCostReporter(cumulWeightedCosts); + tcr2.showInJFrame("Average weighted target costs", false, false); + tcr2.start(); + } + } + + protected void rememberWhichWeightsAreNonZero() { + // remember which weights are non-zero + weightsNonZero = new boolean[featureDefinition.getNumberOfFeatures()]; + for (int i = 0, n = featureDefinition.getNumberOfFeatures(); i < n; i++) { + weightsNonZero[i] = (featureDefinition.getWeight(i) > 0); + } + } + + /** + * Compute the features for a given target, and store them in the target. + * + * @param target + * the target for which to compute the features + * @see Target#getFeatureVector() + */ + public void computeTargetFeatures(Target target) { + FeatureVector fv = targetFeatureComputer.computeFeatureVector(target); + target.setFeatureVector(fv); + } + + /** + * Look up the features for a given unit. + * + * @param unit + * a unit in the database + * @return the FeatureVector for target cost computation associated to this unit + */ + public FeatureVector getFeatureVector(Unit unit) { + return featureVectors[unit.index]; + } + + /** + * Get the string representation of the feature value associated with the given unit + * + * @param unit + * the unit whose feature value is requested + * @param featureName + * name of the feature requested + * @return a string representation of the feature value + * @throws IllegalArgumentException + * if featureName is not a known feature + */ + public String getFeature(Unit unit, String featureName) { + int featureIndex = featureDefinition.getFeatureIndex(featureName); + if (featureDefinition.isByteFeature(featureIndex)) { + byte value = featureVectors[unit.index].getByteFeature(featureIndex); + return featureDefinition.getFeatureValueAsString(featureIndex, value); + } else if (featureDefinition.isShortFeature(featureIndex)) { + short value = featureVectors[unit.index].getShortFeature(featureIndex); + return featureDefinition.getFeatureValueAsString(featureIndex, value); + } else { // continuous -- return float as string + float value = featureVectors[unit.index].getContinuousFeature(featureIndex); + return String.valueOf(value); + } + } + + public FeatureDefinition getFeatureDefinition() { + return featureDefinition; + } + + public class TargetCostReporter extends Histogram { + private double[] data; + private int lastN = 0; + + public TargetCostReporter(double[] data) { + super(0, 1, data); + this.data = data; + } + + public void start() { + new Thread() { + public void run() { + while (isVisible()) { + try { + Thread.sleep(500); + } catch (InterruptedException ie) { + } + updateGraph(); + } + } + }.start(); + } + + protected void updateGraph() { + if (nCostComputations == lastN) + return; + lastN = nCostComputations; + double[] newCosts = new double[data.length]; + for (int i = 0; i < newCosts.length; i++) { + newCosts[i] = data[i] / nCostComputations; + } + updateData(0, 1, newCosts); + repaint(); + } + } + + public FeatureVector[] getFeatureVectors() { + return featureVectors; + } + +} diff --git a/marytts-unitselection/src/main/java/marytts/unitselection/select/HalfPhoneFFRTargetCostFunction.java b/marytts-unitselection/src/main/java/marytts/unitselection/select/HalfPhoneFFRTargetCostFunction.java new file mode 100644 index 0000000000..2e0453cb96 --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/unitselection/select/HalfPhoneFFRTargetCostFunction.java @@ -0,0 +1,245 @@ +/** + * Copyright 2006 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.unitselection.select; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStreamReader; + +import marytts.exceptions.MaryConfigurationException; +import marytts.features.FeatureDefinition; +import marytts.features.FeatureProcessorManager; +import marytts.features.FeatureVector; +import marytts.features.TargetFeatureComputer; +import marytts.features.Target; +import marytts.features.HalfPhoneTarget; +import marytts.server.MaryProperties; +import marytts.signalproc.display.Histogram; +import marytts.unitselection.data.FeatureFileReader; +import marytts.unitselection.data.HalfPhoneFeatureFileReader; +import marytts.unitselection.data.Unit; +import marytts.unitselection.weightingfunctions.WeightFunc; +import marytts.unitselection.weightingfunctions.WeightFunctionManager; +import marytts.util.MaryUtils; + +public class HalfPhoneFFRTargetCostFunction extends FFRTargetCostFunction { + protected FeatureDefinition leftWeights; + protected FeatureDefinition rightWeights; + protected WeightFunc[] leftWeightFunction; + protected WeightFunc[] rightWeightFunction; + + public HalfPhoneFFRTargetCostFunction() { + } + + /** + * Compute the goodness-of-fit of a given unit for a given target. + * + * @param target + * @param unit + * @return a non-negative number; smaller values mean better fit, i.e. smaller cost. + */ + public double cost(Target target, Unit unit) { + if (!(target instanceof HalfPhoneTarget)) + throw new IllegalArgumentException("This target cost function can only be called for half-phone targets!"); + HalfPhoneTarget hpTarget = (HalfPhoneTarget) target; + boolean isLeftHalf = hpTarget.isLeftHalf(); + FeatureDefinition weights = isLeftHalf ? leftWeights : rightWeights; + WeightFunc[] weightFunctions = isLeftHalf ? leftWeightFunction : rightWeightFunction; + return cost(target, unit, weights, weightFunctions); + } + + /** + * Initialise the data needed to do a target cost computation. + * + * @param featureFileName + * name of a file containing the unit features + * @param weightsFile + * an optional string containing weights file names -- if non-null, contains two file names separated by the + * character '|', pointing to feature weights files for left and right units, respectively, that override the ones + * present in the feature file. + * @param featProc + * a feature processor manager which can provide feature processors to compute the features for a target at run + * time + * @throws IOException + */ + public void load(String featureFileName, String weightsFile, FeatureProcessorManager featProc) throws IOException, + MaryConfigurationException { + HalfPhoneFeatureFileReader ffr = new HalfPhoneFeatureFileReader(featureFileName); + load(ffr, weightsFile, featProc); + } + + public void load(FeatureFileReader featureFileReader, String weightsFile, FeatureProcessorManager featProc) + throws IOException { + if (!(featureFileReader instanceof HalfPhoneFeatureFileReader)) + throw new IllegalArgumentException("Featurefilereader must be a HalfPhoneFeatureFileReader"); + HalfPhoneFeatureFileReader ffr = (HalfPhoneFeatureFileReader) featureFileReader; + this.leftWeights = ffr.getLeftWeights(); + this.featureDefinition = this.leftWeights; + this.rightWeights = ffr.getRightWeights(); + this.featureVectors = ffr.getFeatureVectors(); + + if (weightsFile != null) { + MaryUtils.getLogger("TargetCostFeatures").debug("Overwriting target cost weights from file " + weightsFile); + String[] weightsFiles = weightsFile.split("\\|"); + if (weightsFiles.length != 2) + throw new IllegalArgumentException( + "Parameter weightsFile should contain exactly two fields separated by a '|' character -- instead, it is: '" + + weightsFile + "'"); + File leftF = new File(weightsFiles[0].trim()); + File rightF; + // If the second weights file has no path, it is in the same directory as the first + if (weightsFiles[1].indexOf("/") == -1 && weightsFiles[1].indexOf("\\") == -1) { + File dir = leftF.getParentFile(); + rightF = new File(dir, weightsFiles[1].trim()); + } else { + rightF = new File(weightsFiles[1].trim()); + } + + // overwrite weights from files + FeatureDefinition newLeftWeights = new FeatureDefinition(new BufferedReader(new InputStreamReader( + new FileInputStream(leftF), "UTF-8")), true); + if (!newLeftWeights.featureEquals(leftWeights)) { + throw new IOException("Weights file '" + leftF + "': feature definition incompatible with feature file"); + } + leftWeights = newLeftWeights; + FeatureDefinition newRightWeights = new FeatureDefinition(new BufferedReader(new InputStreamReader( + new FileInputStream(rightF), "UTF-8")), true); + if (!newRightWeights.featureEquals(rightWeights)) { + throw new IOException("Weights file '" + rightF + "': feature definition incompatible with feature file"); + } + rightWeights = newRightWeights; + } + WeightFunctionManager wfm = new WeightFunctionManager(); + WeightFunc linear = wfm.getWeightFunction("linear"); + int nDiscreteFeatures = leftWeights.getNumberOfByteFeatures() + leftWeights.getNumberOfShortFeatures(); + leftWeightFunction = new WeightFunc[leftWeights.getNumberOfContinuousFeatures()]; + rightWeightFunction = new WeightFunc[leftWeightFunction.length]; + for (int i = 0; i < leftWeightFunction.length; i++) { + String weightFunctionName = leftWeights.getWeightFunctionName(nDiscreteFeatures + i); + if ("".equals(weightFunctionName)) + leftWeightFunction[i] = linear; + else + leftWeightFunction[i] = wfm.getWeightFunction(weightFunctionName); + weightFunctionName = rightWeights.getWeightFunctionName(nDiscreteFeatures + i); + if ("".equals(weightFunctionName)) + rightWeightFunction[i] = linear; + else + rightWeightFunction[i] = wfm.getWeightFunction(weightFunctionName); + } + // TODO: If the target feature computer had direct access to the feature definition, it could do some consistency checking + this.targetFeatureComputer = new TargetFeatureComputer(featProc, leftWeights.getFeatureNames()); + + rememberWhichWeightsAreNonZero(); + + if (MaryProperties.getBoolean("debug.show.cost.graph")) { + debugShowCostGraph = true; + cumulWeightedCosts = new double[featureDefinition.getNumberOfFeatures()]; + TargetCostReporter tcr2 = new TargetCostReporter(cumulWeightedCosts); + tcr2.showInJFrame("Average weighted target costs", false, false); + tcr2.start(); + } + } + + /** + * Compute the features for a given target, and store them in the target. + * + * @param target + * the target for which to compute the features + * @see Target#getFeatureVector() + */ + public void computeTargetFeatures(Target target) { + FeatureVector fv = targetFeatureComputer.computeFeatureVector(target); + target.setFeatureVector(fv); + } + + /** + * Look up the features for a given unit. + * + * @param unit + * a unit in the database + * @return the FeatureVector for target cost computation associated to this unit + */ + public FeatureVector getUnitFeatures(Unit unit) { + return featureVectors[unit.index]; + } + + /** + * Get the string representation of the feature value associated with the given unit + * + * @param unit + * the unit whose feature value is requested + * @param featureName + * name of the feature requested + * @return a string representation of the feature value + * @throws IllegalArgumentException + * if featureName is not a known feature + */ + public String getFeature(Unit unit, String featureName) { + int featureIndex = featureDefinition.getFeatureIndex(featureName); + if (featureDefinition.isByteFeature(featureIndex)) { + byte value = featureVectors[unit.index].getByteFeature(featureIndex); + return featureDefinition.getFeatureValueAsString(featureIndex, value); + } else if (featureDefinition.isShortFeature(featureIndex)) { + short value = featureVectors[unit.index].getShortFeature(featureIndex); + return featureDefinition.getFeatureValueAsString(featureIndex, value); + } else { // continuous -- return float as string + float value = featureVectors[unit.index].getContinuousFeature(featureIndex); + return String.valueOf(value); + } + } + + public class TargetCostReporter extends Histogram { + private double[] data; + private int lastN = 0; + + public TargetCostReporter(double[] data) { + super(0, 1, data); + this.data = data; + } + + public void start() { + new Thread() { + public void run() { + while (isVisible()) { + try { + Thread.sleep(500); + } catch (InterruptedException ie) { + } + updateGraph(); + } + } + }.start(); + } + + protected void updateGraph() { + if (nCostComputations == lastN) + return; + lastN = nCostComputations; + double[] newCosts = new double[data.length]; + for (int i = 0; i < newCosts.length; i++) { + newCosts[i] = data[i] / nCostComputations; + } + updateData(0, 1, newCosts); + repaint(); + } + } +} diff --git a/marytts-unitselection/src/main/java/marytts/unitselection/select/HalfPhoneUnitSelector.java b/marytts-unitselection/src/main/java/marytts/unitselection/select/HalfPhoneUnitSelector.java new file mode 100644 index 0000000000..55777f3f28 --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/unitselection/select/HalfPhoneUnitSelector.java @@ -0,0 +1,59 @@ +/** + * Copyright 2006 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.unitselection.select; + +import java.util.ArrayList; +import java.util.List; + +import marytts.unitselection.data.UnitDatabase; +import marytts.features.Target; +import marytts.features.HalfPhoneTarget; + +import org.w3c.dom.Element; + +public class HalfPhoneUnitSelector extends UnitSelector { + + /** + * Initialise the unit selector. Need to call load() separately. + * + * @see #load(UnitDatabase) + */ + public HalfPhoneUnitSelector() throws Exception { + super(); + } + + /** + * Create the list of targets from the XML elements to synthesize. + * + * @param segmentsAndBoundaries + * a list of MaryXML phone and boundary elements + * @return a list of Target objects + */ + protected List createTargets(List segmentsAndBoundaries) { + List targets = new ArrayList(); + for (Element sOrB : segmentsAndBoundaries) { + String phone = getPhoneSymbol(sOrB); + targets.add(new HalfPhoneTarget(phone + "_L", sOrB, true)); // left half + targets.add(new HalfPhoneTarget(phone + "_R", sOrB, false)); // right half + } + return targets; + } + +} diff --git a/marytts-unitselection/src/main/java/marytts/unitselection/select/JoinCostFeatures.java b/marytts-unitselection/src/main/java/marytts/unitselection/select/JoinCostFeatures.java new file mode 100644 index 0000000000..0052a93134 --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/unitselection/select/JoinCostFeatures.java @@ -0,0 +1,626 @@ +/** + * Copyright 2006 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.unitselection.select; + +import java.io.BufferedInputStream; +import java.io.BufferedReader; +import java.io.DataInput; +import java.io.DataInputStream; +import java.io.EOFException; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.ByteBuffer; +import java.nio.FloatBuffer; +import java.nio.channels.FileChannel; +import java.util.Vector; + +import marytts.exceptions.MaryConfigurationException; +import marytts.features.ByteValuedFeatureProcessor; +import marytts.features.MaryGenericFeatureProcessors; +import marytts.features.Target; +import marytts.features.DiphoneTarget; +import marytts.features.HalfPhoneTarget; +import marytts.modules.phonemiser.Allophone; +import marytts.server.MaryProperties; +import marytts.signalproc.display.Histogram; +import marytts.unitselection.data.DiphoneUnit; +import marytts.unitselection.data.Unit; +import marytts.unitselection.weightingfunctions.WeightFunc; +import marytts.unitselection.weightingfunctions.WeightFunctionManager; +import marytts.util.MaryUtils; +import marytts.util.data.MaryHeader; +import marytts.util.io.StreamUtils; + +public class JoinCostFeatures implements JoinCostFunction { + + protected float wSignal; + protected float wPhonetic; + + protected boolean debugShowCostGraph = false; + protected double[] cumulWeightedSignalCosts = null; + protected int nCostComputations = 0; + + protected PrecompiledJoinCostReader precompiledCosts; + + protected JoinCostReporter jcr; + + /****************/ + /* DATA FIELDS */ + /****************/ + private MaryHeader hdr = null; + + private float[] featureWeight = null; + private WeightFunc[] weightFunction = null; + private boolean[] isLinear = null; // wether the i'th weight function is a linear function + + private float[][] leftJCF = null; + private float[][] rightJCF = null; + + /****************/ + /* CONSTRUCTORS */ + /****************/ + + /** + * Empty constructor; when using this, call load() separately to initialise this class. + * + * @see #load(String) + */ + public JoinCostFeatures() { + } + + /** + * Constructor which read a Mary Join Cost file. + */ + public JoinCostFeatures(String fileName) throws IOException, MaryConfigurationException { + load(fileName, null, null, (float) 0.5); + } + + /** + * Initialise this join cost function by reading the appropriate settings from the MaryProperties using the given + * configPrefix. + * + * @param configPrefix + * the prefix for the (voice-specific) config entries to use when looking up files to load. + */ + public void init(String configPrefix) throws MaryConfigurationException { + String joinFileName = MaryProperties.needFilename(configPrefix + ".joinCostFile"); + String precomputedJoinCostFileName = MaryProperties.getFilename(configPrefix + ".precomputedJoinCostFile"); + float wSignal = Float.parseFloat(MaryProperties.getProperty(configPrefix + ".joincostfunction.wSignal", "1.0")); + try { + InputStream joinWeightStream = MaryProperties.getStream(configPrefix + ".joinCostWeights"); + load(joinFileName, joinWeightStream, precomputedJoinCostFileName, wSignal); + } catch (IOException ioe) { + throw new MaryConfigurationException("Problem loading join file " + joinFileName, ioe); + } + } + + /** + * Load weights and values from the given file + * + * @param joinFileName + * the file from which to read default weights and join cost features + * @param weightStream + * an optional file from which to read weights, taking precedence over + * @param precompiledCostFileName + * an optional file containing precompiled join costs + * @param wSignal + * Relative weight of the signal-based join costs relative to the phonetic join costs computed from the target + */ + public void load(String joinFileName, InputStream weightStream, String precompiledCostFileName, float wSignal) + throws IOException, MaryConfigurationException { + loadFromByteBuffer(joinFileName, weightStream, precompiledCostFileName, wSignal); + } + + /** + * Load weights and values from the given file + * + * @param joinFileName + * the file from which to read default weights and join cost features + * @param weightStream + * an optional file from which to read weights, taking precedence over + * @param precompiledCostFileName + * an optional file containing precompiled join costs + * @param wSignal + * Relative weight of the signal-based join costs relative to the phonetic join costs computed from the target + */ + private void loadFromByteBuffer(String joinFileName, InputStream weightStream, String precompiledCostFileName, float wSignal) + throws IOException, MaryConfigurationException { + if (precompiledCostFileName != null) { + precompiledCosts = new PrecompiledJoinCostReader(precompiledCostFileName); + } + this.wSignal = wSignal; + wPhonetic = 1 - wSignal; + /* Open the file */ + FileInputStream fis = new FileInputStream(joinFileName); + FileChannel fc = fis.getChannel(); + ByteBuffer bb = fc.map(FileChannel.MapMode.READ_ONLY, 0, fc.size()); + /* Read the Mary header */ + hdr = new MaryHeader(bb); + if (hdr.getType() != MaryHeader.JOINFEATS) { + throw new IOException("File [" + joinFileName + "] is not a valid Mary join features file."); + } + try { + /* Read the feature weights and feature processors */ + int numberOfFeatures = bb.getInt(); + featureWeight = new float[numberOfFeatures]; + weightFunction = new WeightFunc[numberOfFeatures]; + isLinear = new boolean[numberOfFeatures]; + WeightFunctionManager wfm = new WeightFunctionManager(); + String wfStr = null; + for (int i = 0; i < numberOfFeatures; i++) { + featureWeight[i] = bb.getFloat(); + wfStr = StreamUtils.readUTF(bb); + if ("".equals(wfStr)) + weightFunction[i] = wfm.getWeightFunction("linear"); + else + weightFunction[i] = wfm.getWeightFunction(wfStr); + } + // Overwrite weights and weight functions from file? + if (weightStream != null) { + MaryUtils.getLogger("JoinCostFeatures").debug("Overwriting join cost weights"); + Object[] weightData = readJoinCostWeightsStream(weightStream); + featureWeight = (float[]) weightData[0]; + String[] wf = (String[]) weightData[1]; + if (featureWeight.length != numberOfFeatures) + throw new IllegalArgumentException("Join cost file contains " + numberOfFeatures + + " features, but weight file contains " + featureWeight.length + " feature weights!"); + for (int i = 0; i < numberOfFeatures; i++) { + weightFunction[i] = wfm.getWeightFunction(wf[i]); + } + } + for (int i = 0; i < numberOfFeatures; i++) { + isLinear[i] = weightFunction[i].whoAmI().equals("linear"); + } + + /* Read the left and right Join Cost Features */ + int numberOfUnits = bb.getInt(); + FloatBuffer fb = bb.asFloatBuffer(); + leftJCF = new float[numberOfUnits][]; + rightJCF = new float[numberOfUnits][]; + for (int i = 0; i < numberOfUnits; i++) { + // System.out.println("Reading join features for unit "+i+" out of "+numberOfUnits); + leftJCF[i] = new float[numberOfFeatures]; + fb.get(leftJCF[i]); + rightJCF[i] = new float[numberOfFeatures]; + fb.get(rightJCF[i]); + } + } catch (EOFException e) { + IOException ioe = new IOException("The currently read Join Cost File has prematurely reached EOF."); + ioe.initCause(e); + throw ioe; + + } + if (MaryProperties.getBoolean("debug.show.cost.graph")) { + debugShowCostGraph = true; + cumulWeightedSignalCosts = new double[featureWeight.length]; + jcr = new JoinCostReporter(cumulWeightedSignalCosts); + jcr.showInJFrame("Average signal join costs", false, false); + jcr.start(); + } + + } + + /** + * Load weights and values from the given file + * + * @param joinFileName + * the file from which to read default weights and join cost features + * @param weightStream + * an optional file from which to read weights, taking precedence over + * @param precompiledCostFileName + * an optional file containing precompiled join costs + * @param wSignal + * Relative weight of the signal-based join costs relative to the phonetic join costs computed from the target + */ + private void loadFromStream(String joinFileName, InputStream weightStream, String precompiledCostFileName, float wSignal) + throws IOException, MaryConfigurationException { + if (precompiledCostFileName != null) { + precompiledCosts = new PrecompiledJoinCostReader(precompiledCostFileName); + } + this.wSignal = wSignal; + wPhonetic = 1 - wSignal; + /* Open the file */ + File fid = new File(joinFileName); + DataInput raf = new DataInputStream(new BufferedInputStream(new FileInputStream(fid))); + /* Read the Mary header */ + hdr = new MaryHeader(raf); + if (hdr.getType() != MaryHeader.JOINFEATS) { + throw new MaryConfigurationException("File [" + joinFileName + "] is not a valid Mary join features file."); + } + try { + /* Read the feature weights and feature processors */ + int numberOfFeatures = raf.readInt(); + featureWeight = new float[numberOfFeatures]; + weightFunction = new WeightFunc[numberOfFeatures]; + isLinear = new boolean[numberOfFeatures]; + WeightFunctionManager wfm = new WeightFunctionManager(); + String wfStr = null; + for (int i = 0; i < numberOfFeatures; i++) { + featureWeight[i] = raf.readFloat(); + wfStr = raf.readUTF(); + if ("".equals(wfStr)) + weightFunction[i] = wfm.getWeightFunction("linear"); + else + weightFunction[i] = wfm.getWeightFunction(wfStr); + } + // Overwrite weights and weight functions from file? + if (weightStream != null) { + MaryUtils.getLogger("JoinCostFeatures").debug("Overwriting join cost weights"); + Object[] weightData = readJoinCostWeightsStream(weightStream); + featureWeight = (float[]) weightData[0]; + String[] wf = (String[]) weightData[1]; + if (featureWeight.length != numberOfFeatures) + throw new IllegalArgumentException("Join cost file contains " + numberOfFeatures + + " features, but weight file contains " + featureWeight.length + " feature weights!"); + for (int i = 0; i < numberOfFeatures; i++) { + weightFunction[i] = wfm.getWeightFunction(wf[i]); + } + } + for (int i = 0; i < numberOfFeatures; i++) { + isLinear[i] = weightFunction[i].whoAmI().equals("linear"); + } + + /* Read the left and right Join Cost Features */ + int numberOfUnits = raf.readInt(); + leftJCF = new float[numberOfUnits][]; + rightJCF = new float[numberOfUnits][]; + for (int i = 0; i < numberOfUnits; i++) { + // System.out.println("Reading join features for unit "+i+" out of "+numberOfUnits); + leftJCF[i] = new float[numberOfFeatures]; + for (int j = 0; j < numberOfFeatures; j++) { + leftJCF[i][j] = raf.readFloat(); + } + rightJCF[i] = new float[numberOfFeatures]; + for (int j = 0; j < numberOfFeatures; j++) { + rightJCF[i][j] = raf.readFloat(); + } + } + } catch (EOFException e) { + IOException ioe = new IOException("The currently read Join Cost File has prematurely reached EOF."); + ioe.initCause(e); + throw ioe; + + } + if (MaryProperties.getBoolean("debug.show.cost.graph")) { + debugShowCostGraph = true; + cumulWeightedSignalCosts = new double[featureWeight.length]; + jcr = new JoinCostReporter(cumulWeightedSignalCosts); + jcr.showInJFrame("Average signal join costs", false, false); + jcr.start(); + } + + } + + /** + * Read the join cost weight specifications from the given file. The weights will be normalized such that they sum to one. + * + * @param fileName + * the text file containing the join weights + * */ + public static Object[] readJoinCostWeightsFile(String fileName) throws IOException, FileNotFoundException { + return readJoinCostWeightsStream(new FileInputStream(fileName)); + } + + /** + * Read the join cost weight specifications from the given file. The weights will be normalized such that they sum to one. + * + * @param fileName + * the text file containing the join weights + * */ + public static Object[] readJoinCostWeightsStream(InputStream weightStream) throws IOException, FileNotFoundException { + Vector v = new Vector(16, 16); + Vector vf = new Vector(16, 16); + /* Open the file */ + BufferedReader in = new BufferedReader(new InputStreamReader(weightStream, "UTF-8")); + /* Loop through the lines */ + String line = null; + String[] fields = null; + float sumOfWeights = 0; + while ((line = in.readLine()) != null) { + // System.out.println( line ); + line = line.split("#", 2)[0]; // Remove possible trailing comments + line = line.trim(); // Remove leading and trailing blanks + if (line.equals("")) + continue; // Empty line: don't parse + line = line.split(":", 2)[1].trim(); // Remove the line number and : + // System.out.print( "CLEANED: [" + line + "]" ); + fields = line.split("\\s", 2); // Separate the weight value from the function name + float aWeight = Float.parseFloat(fields[0]); + sumOfWeights += aWeight; + v.add(new Float(aWeight)); // Push the weight + vf.add(fields[1]); // Push the function + // System.out.println( "NBFEA=" + numberOfFeatures ); + } + in.close(); + // System.out.flush(); + /* Export the vector of weighting function names as a String array: */ + String[] wfun = (String[]) vf.toArray(new String[vf.size()]); + /* + * For the weights, create a float array containing the weights, normalized such that they sum to one: + */ + float[] fw = new float[v.size()]; + for (int i = 0; i < fw.length; i++) { + Float aWeight = (Float) v.get(i); + fw[i] = aWeight.floatValue() / sumOfWeights; + } + /* Return these as an Object[2]. */ + return new Object[] { fw, wfun }; + } + + /*****************/ + /* ACCESSORS */ + /*****************/ + + /** + * Get the number of feature weights and weighting functions. + */ + public int getNumberOfFeatures() { + return (featureWeight.length); + } + + /** + * Get the number of units. + */ + public int getNumberOfUnits() { + return (leftJCF.length); + } + + /** + * Gets the array of left join cost features for a particular unit index. + * + * @param u + * The index of the considered unit. + * + * @return The array of left join cost features for the given unit. + */ + public float[] getLeftJCF(int u) { + if (u < 0) { + throw new RuntimeException("The unit index [" + u + "] is out of range: a unit index can't be negative."); + } + if (u > getNumberOfUnits()) { + throw new RuntimeException("The unit index [" + u + "] is out of range: this file contains [" + getNumberOfUnits() + + "] units."); + } + return (leftJCF[u]); + } + + /** + * Gets the array of right join cost features for a particular unit index. + * + * @param u + * The index of the considered unit. + * + * @return The array of right join cost features for the given unit. + */ + public float[] getRightJCF(int u) { + if (u < 0) { + throw new RuntimeException("The unit index [" + u + "] is out of range: a unit index can't be negative."); + } + if (u > getNumberOfUnits()) { + throw new RuntimeException("The unit index [" + u + "] is out of range: this file contains [" + getNumberOfUnits() + + "] units."); + } + return (rightJCF[u]); + } + + /*****************/ + /* MISC METHODS */ + /*****************/ + + /** + * Deliver the join cost between two units described by their index. + * + * @param u1 + * the left unit + * @param u2 + * the right unit + * + * @return the cost of joining the right Join Cost features of the left unit with the left Join Cost Features of the right + * unit. + */ + public double cost(int u1, int u2) { + /* Check the given indexes */ + if (u1 < 0) { + throw new RuntimeException("The left unit index [" + u1 + "] is out of range: a unit index can't be negative."); + } + // if ( u1 > getNumberOfUnits() ) { + if (u1 > leftJCF.length) { + throw new RuntimeException("The left unit index [" + u1 + "] is out of range: this file contains [" + + getNumberOfUnits() + "] units."); + } + if (u2 < 0) { + throw new RuntimeException("The right unit index [" + u2 + "] is out of range: a unit index can't be negative."); + } + // if ( u2 > getNumberOfUnits() ) { + if (u2 > leftJCF.length) { + throw new RuntimeException("The right unit index [" + u2 + "] is out of range: this file contains [" + + getNumberOfUnits() + "] units."); + } + if (debugShowCostGraph) { + jcr.tick(); + } + /* Cumulate the join costs for each feature */ + double res = 0.0; + float[] v1 = rightJCF[u1]; + float[] v2 = leftJCF[u2]; + for (int i = 0; i < v1.length; i++) { + float a = v1[i]; + float b = v2[i]; + // if (!Float.isNaN(v1[i]) && !Float.isNaN(v2[i])) { + if (!(a != a) && !(b != b)) { + double c; + if (isLinear[i]) { + c = featureWeight[i] * (a > b ? (a - b) : (b - a)); + } else { + c = featureWeight[i] * weightFunction[i].cost(a, b); + } + res += c; + if (debugShowCostGraph) { + cumulWeightedSignalCosts[i] += wSignal * c; + } + } // if anything is NaN, count the cost as 0. + } + return (res); + } + + /** + * A combined cost computation, as a weighted sum of the signal-based cost (computed from the units) and the phonetics-based + * cost (computed from the targets). + * + * @param t1 + * The left target. + * @param u1 + * The left unit. + * @param t2 + * The right target. + * @param u2 + * The right unit. + * + * @return the cost of joining the left unit with the right unit, as a non-negative value. + */ + public double cost(Target t1, Unit u1, Target t2, Unit u2) { + // Units of length 0 cannot be joined: + if (u1.duration == 0 || u2.duration == 0) + return Double.POSITIVE_INFINITY; + // In the case of diphones, replace them with the relevant part: + boolean bothDiphones = true; + if (u1 instanceof DiphoneUnit) { + u1 = ((DiphoneUnit) u1).right; + } else { + bothDiphones = false; + } + if (u2 instanceof DiphoneUnit) { + u2 = ((DiphoneUnit) u2).left; + } else { + bothDiphones = false; + } + + if (u1.index + 1 == u2.index) + return 0; + // Either not half phone synthesis, or at a diphone boundary + double cost = 1; // basic penalty for joins of non-contiguous units. + if (bothDiphones && precompiledCosts != null) { + cost += precompiledCosts.cost(t1, u1, t2, u2); + } else { // need to actually compute the cost + cost += cost(u1.index, u2.index); + } + return cost; + } + + /** + * A phonetic join cost, computed solely from the target. + * + * @param t1 + * the left target + * @param t2 + * the right target + * @return a non-negative join cost, usually between 0 (best) and 1 (worst). + * @deprecated + */ + protected double cost(Target t1, Target t2) { + // TODO: This is really ad hoc for the moment. Redo once we know what we are doing. + // Add penalties for a number of criteria. + double cost = 0; + ByteValuedFeatureProcessor stressProcessor = new MaryGenericFeatureProcessors.Stressed("", + new MaryGenericFeatureProcessors.SyllableNavigator()); + // Stressed? + boolean stressed1 = stressProcessor.process(t1) == (byte) 1; + boolean stressed2 = stressProcessor.process(t1) == (byte) 1; + // Try to avoid joining in a stressed syllable: + if (stressed1 || stressed2) + cost += 0.2; + Allophone p1 = t1.getAllophone(); + Allophone p2 = t2.getAllophone(); + + // Discourage joining vowels: + if (p1.isVowel() || p2.isVowel()) + cost += 0.2; + // Discourage joining glides: + if (p1.isGlide() || p2.isGlide()) + cost += 0.2; + // Discourage joining voiced segments: + if (p1.isVoiced() || p2.isVoiced()) + cost += 0.1; + // If both are voiced, it's really bad + if (p1.isVoiced() && p2.isVoiced()) + cost += 0.1; + // Slightly penalize nasals and liquids + if (p1.isNasal() || p2.isNasal()) + cost += 0.05; + if (p1.isLiquid() || p2.isLiquid()) + cost += 0.05; + // Fricatives -- nothing? + // Plosives -- nothing? + + if (cost > 1) + cost = 1; + return cost; + } + + public static class JoinCostReporter extends Histogram { + private double[] data; + private int lastN = 0; + private int nCostComputations = 0; + + public JoinCostReporter(double[] data) { + super(0, 1, data); + this.data = data; + } + + public void start() { + new Thread() { + public void run() { + while (isVisible()) { + try { + Thread.sleep(500); + } catch (InterruptedException ie) { + } + updateGraph(); + } + } + }.start(); + } + + /** + * Register one new cost computation + */ + public void tick() { + nCostComputations++; + } + + protected void updateGraph() { + if (nCostComputations == lastN) + return; + lastN = nCostComputations; + double[] newCosts = new double[data.length]; + for (int i = 0; i < newCosts.length; i++) { + newCosts[i] = data[i] / nCostComputations; + } + updateData(0, 1, newCosts); + repaint(); + } + } + +} diff --git a/marytts-unitselection/src/main/java/marytts/unitselection/select/JoinCostFunction.java b/marytts-unitselection/src/main/java/marytts/unitselection/select/JoinCostFunction.java new file mode 100644 index 0000000000..4c59e36e34 --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/unitselection/select/JoinCostFunction.java @@ -0,0 +1,78 @@ +/** + * Copyright 2006 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.unitselection.select; + +import java.io.IOException; +import java.io.InputStream; + +import marytts.exceptions.MaryConfigurationException; +import marytts.unitselection.data.Unit; +import marytts.features.Target; + +/** + * A join cost function for evaluating the goodness-of-fit of a given pair of left and right unit. + * + * @author Marc Schröder + * + */ +public interface JoinCostFunction { + /** + * Compute the goodness-of-fit of joining two units, given the corresponding targets + * + * @param t1 + * the left target + * @param u1 + * the proposed left unit + * @param t2 + * the right target + * @param u3 + * the proposed right unit + * @return a non-negative number; smaller values mean better fit, i.e. smaller cost. + */ + public double cost(Target t1, Unit u1, Target t2, Unit u2); + + /** + * Initialise this join cost function by reading the appropriate settings from the MaryProperties using the given + * configPrefix. + * + * @param configPrefix + * the prefix for the (voice-specific) config entries to use when looking up files to load. + * @throws MaryConfigurationException + * if there is a configuration problem + */ + public void init(String configPrefix) throws MaryConfigurationException; + + /** + * Load weights and values from the given file + * + * @param joinFileName + * the file from which to read default weights and join cost features + * @param weightStream + * an optional file from which to read weights, taking precedence over + * @param precompiledCostFileName + * an optional file containing precompiled join costs + * @param wSignal + * Relative weight of the signal-based join costs relative to the phonetic join costs computed from the target + */ + @Deprecated + public void load(String joinFileName, InputStream weightStream, String precompiledCostFileName, float wSignal) + throws IOException, MaryConfigurationException; + +} diff --git a/marytts-unitselection/src/main/java/marytts/unitselection/select/JoinModelCost.java b/marytts-unitselection/src/main/java/marytts/unitselection/select/JoinModelCost.java new file mode 100644 index 0000000000..379b243f16 --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/unitselection/select/JoinModelCost.java @@ -0,0 +1,220 @@ +/** + * Copyright 2006 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.unitselection.select; + +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; + +import marytts.cart.CART; +import marytts.cart.Node; +import marytts.cart.LeafNode.PdfLeafNode; +import marytts.cart.io.HTSCARTReader; +import marytts.exceptions.MaryConfigurationException; +import marytts.features.FeatureDefinition; +import marytts.features.FeatureVector; +import marytts.features.Target; +import marytts.features.DiphoneTarget; +import marytts.features.HalfPhoneTarget; +import marytts.htsengine.PhoneTranslator; +import marytts.htsengine.HMMData.PdfFileFormat; +import marytts.server.MaryProperties; +import marytts.signalproc.analysis.distance.DistanceComputer; +import marytts.unitselection.data.DiphoneUnit; +import marytts.unitselection.data.Unit; + +public class JoinModelCost implements JoinCostFunction { + protected int nCostComputations = 0; + + /****************/ + /* DATA FIELDS */ + /****************/ + private JoinCostFeatures jcf = null; + + CART[] joinTree = null; // an array of carts, one per HMM state. + + private float f0Weight; + + private FeatureDefinition featureDef = null; + + private boolean debugShowCostGraph = false; + + /****************/ + /* CONSTRUCTORS */ + /****************/ + + /** + * Empty constructor; when using this, call load() separately to initialise this class. + * + * @see #load(String) + */ + public JoinModelCost() { + } + + /** + * Initialise this join cost function by reading the appropriate settings from the MaryProperties using the given + * configPrefix. + * + * @param configPrefix + * the prefix for the (voice-specific) config entries to use when looking up files to load. + */ + public void init(String configPrefix) throws MaryConfigurationException { + try { + String joinFileName = MaryProperties.needFilename(configPrefix + ".joinCostFile"); + InputStream joinPdfStream = MaryProperties.needStream(configPrefix + ".joinPdfFile"); + InputStream joinTreeStream = MaryProperties.needStream(configPrefix + ".joinTreeFile"); + // CHECK not tested the trickyPhonesFile needs to be added into the configuration file + String trickyPhonesFileName = MaryProperties.needFilename(configPrefix + ".trickyPhonesFile"); + load(joinFileName, joinPdfStream, joinTreeStream, trickyPhonesFileName); + } catch (IOException ioe) { + throw new MaryConfigurationException("Problem loading join file", ioe); + } + } + + @Override + @Deprecated + public void load(String a, InputStream b, String c, float d) { + throw new RuntimeException("Do not use load() -- use init()"); + } + + /** + * Load weights and values from the given file + * + * @param joinFileName + * the file from which to read join cost features + * @param joinPdfFileName + * the file from which to read the Gaussian models in the leaves of the tree + * @param joinTreeFileName + * the file from which to read the Tree, in HTS format. + */ + public void load(String joinFileName, InputStream joinPdfStream, InputStream joinTreeStream, String trickyPhonesFile) + throws IOException, MaryConfigurationException { + jcf = new JoinCostFeatures(joinFileName); + + assert featureDef != null : "Expected to have a feature definition, but it is null!"; + + /* Load Trees */ + HTSCARTReader htsReader = new HTSCARTReader(); + int numStates = 1; // just one state in the joinModeller + + // Check if there are tricky phones, and create a PhoneTranslator object + PhoneTranslator phTranslator = new PhoneTranslator(new FileInputStream(trickyPhonesFile)); + + try { + // joinTree.loadTreeSetGeneral(joinTreeFileName, 0, featureDef); + joinTree = htsReader.load(numStates, joinTreeStream, joinPdfStream, PdfFileFormat.join, featureDef, phTranslator); + + } catch (Exception e) { + IOException ioe = new IOException("Cannot load join model trees"); + ioe.initCause(e); + throw ioe; + } + + } + + /** + * Set the feature definition to use for interpreting target feature vectors. + * + * @param def + * the feature definition to use. + */ + public void setFeatureDefinition(FeatureDefinition def) { + this.featureDef = def; + } + + /*****************/ + /* MISC METHODS */ + /*****************/ + + /** + * A combined cost computation, as a weighted sum of the signal-based cost (computed from the units) and the phonetics-based + * cost (computed from the targets). + * + * @param t1 + * The left target. + * @param u1 + * The left unit. + * @param t2 + * The right target. + * @param u2 + * The right unit. + * + * @return the cost of joining the left unit with the right unit, as a non-negative value. + */ + public double cost(Target t1, Unit u1, Target t2, Unit u2) { + // Units of length 0 cannot be joined: + if (u1.duration == 0 || u2.duration == 0) + return Double.POSITIVE_INFINITY; + // In the case of diphones, replace them with the relevant part: + if (u1 instanceof DiphoneUnit) { + u1 = ((DiphoneUnit) u1).right; + } + if (u2 instanceof DiphoneUnit) { + u2 = ((DiphoneUnit) u2).left; + } + + if (u1.index + 1 == u2.index) + return 0; + double cost = 1; // basic penalty for joins of non-contiguous units. + + float[] v1 = jcf.getRightJCF(u1.index); + float[] v2 = jcf.getLeftJCF(u2.index); + // double[] diff = new double[v1.length]; + // for ( int i = 0; i < v1.length; i++ ) { + // diff[i] = (double)v1[i] - v2[i]; + // } + double[] diff = new double[v1.length]; + for (int i = 0; i < v1.length; i++) { + diff[i] = (double) v1[i] - v2[i]; + } + + // Now evaluate likelihood of the diff under the join model + // Compute the model name: + assert featureDef != null : "Feature Definition was not set"; + FeatureVector fv1 = null; + if (t1 instanceof DiphoneTarget) { + HalfPhoneTarget hpt1 = ((DiphoneTarget) t1).right; + assert hpt1 != null; + fv1 = hpt1.getFeatureVector(); + } else { + fv1 = t1.getFeatureVector(); + } + assert fv1 != null : "Target has no feature vector"; + // String modelName = contextTranslator.features2context(featureDef, fv1, featureList); + + int state = 0; // just one state in the joinModeller + double[] mean; + double[] variance; + + Node node = joinTree[state].interpretToNode(fv1, 1); + + assert node instanceof PdfLeafNode : "The node must be a PdfLeafNode."; + + mean = ((PdfLeafNode) node).getMean(); + variance = ((PdfLeafNode) node).getVariance(); + + double distance = DistanceComputer.getNormalizedEuclideanDistance(diff, mean, variance); + + cost += distance; + + return cost; + } + +} diff --git a/marytts-unitselection/src/main/java/marytts/unitselection/select/PrecompiledJoinCostReader.java b/marytts-unitselection/src/main/java/marytts/unitselection/select/PrecompiledJoinCostReader.java new file mode 100644 index 0000000000..c54a85ba91 --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/unitselection/select/PrecompiledJoinCostReader.java @@ -0,0 +1,159 @@ +/** + * Portions Copyright 2006 DFKI GmbH. + * Portions Copyright 2001 Sun Microsystems, Inc. + * Portions Copyright 1999-2001 Language Technologies Institute, + * Carnegie Mellon University. + * All Rights Reserved. Use is subject to license terms. + * + * Permission is hereby granted, free of charge, to use and distribute + * this software and its documentation without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of this work, and to + * permit persons to whom this work is furnished to do so, subject to + * the following conditions: + * + * 1. The code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * 2. Any modifications must be clearly marked as such. + * 3. Original authors' names are not deleted. + * 4. The authors' names are not used to endorse or promote products + * derived from this software without specific prior written + * permission. + * + * DFKI GMBH AND THE CONTRIBUTORS TO THIS WORK DISCLAIM ALL WARRANTIES WITH + * REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL DFKI GMBH NOR THE + * CONTRIBUTORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL + * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR + * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS + * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF + * THIS SOFTWARE. + */ +package marytts.unitselection.select; + +import java.io.BufferedInputStream; +import java.io.DataInputStream; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.HashMap; +import java.util.Map; + +import marytts.exceptions.MaryConfigurationException; +import marytts.server.MaryProperties; +import marytts.unitselection.data.Unit; +import marytts.util.data.MaryHeader; +import marytts.features.Target; +import marytts.features.DiphoneTarget; +import marytts.features.HalfPhoneTarget; + +/** + * Loads a precompiled join cost file and provides access to the join cost. + * + */ +public class PrecompiledJoinCostReader implements JoinCostFunction { + + private MaryHeader hdr = null; + + // keys = Integers representing left unit index; + // values = maps containing + // - keys = Integers representing right unit index; + // - values = Floats representing the jost of joining them. + protected Map left; + + /** + * Empty constructor; need to call load() separately. + * + * @see #load(String) + */ + public PrecompiledJoinCostReader() { + } + + /** + * Create a precompiled join cost file reader from the given file + * + * @param fileName + * the file to read + * @throws IOException + * if a problem occurs while reading + */ + public PrecompiledJoinCostReader(String fileName) throws IOException, MaryConfigurationException { + load(fileName, null, null, 0); + } + + /** + * Initialise this join cost function by reading the appropriate settings from the MaryProperties using the given + * configPrefix. + * + * @param configPrefix + * the prefix for the (voice-specific) config entries to use when looking up files to load. + */ + public void init(String configPrefix) throws MaryConfigurationException { + String precomputedJoinCostFileName = MaryProperties.getFilename(configPrefix + ".precomputedJoinCostFile"); + try { + load(precomputedJoinCostFileName, null, null, 0); + } catch (IOException ioe) { + throw new MaryConfigurationException("Problem loading join file " + precomputedJoinCostFileName, ioe); + } + } + + /** + * Load the given precompiled join cost file + * + * @param fileName + * the file to read + * @param dummy + * not used, just used to fulfil the join cost function interface + * @param dummy2 + * not used, just used to fulfil the join cost function interface + * @throws IOException + * if a problem occurs while reading + */ + @Override + public void load(String fileName, InputStream dummy, String dummy2, float dummy3) throws IOException, + MaryConfigurationException { + /* Open the file */ + DataInputStream dis = new DataInputStream(new BufferedInputStream(new FileInputStream(fileName))); + hdr = new MaryHeader(dis); + if (hdr.getType() != MaryHeader.PRECOMPUTED_JOINCOSTS) { + throw new MaryConfigurationException("File [" + fileName + "] is not a valid Mary precompiled join costs file."); + } + /* Read the number of units */ + int numberOfLeftUnits = dis.readInt(); + if (numberOfLeftUnits < 0) { + throw new MaryConfigurationException("File [" + fileName + "] has a negative number of units. Aborting."); + } + + left = new HashMap(); + /* Read the start times and durations */ + for (int i = 0; i < numberOfLeftUnits; i++) { + int leftIndex = dis.readInt(); + int numberOfRightUnits = dis.readInt(); + Map right = new HashMap(); + left.put(new Integer(leftIndex), right); + for (int j = 0; j < numberOfRightUnits; j++) { + int rightIndex = dis.readInt(); + float cost = dis.readFloat(); + right.put(new Integer(rightIndex), new Float(cost)); + } + } + + } + + /** + * Return the (precomputed) cost of joining the two given units; if there is no precomputed cost, return + * Double.POSITIVE_INFINITY. + */ + public double cost(Target t1, Unit uleft, Target t2, Unit uright) { + Integer leftIndex = new Integer(uleft.index); + Map rightUnitsMap = (Map) left.get(leftIndex); + if (rightUnitsMap == null) + return Double.POSITIVE_INFINITY; + Integer rightIndex = new Integer(uright.index); + Float cost = (Float) rightUnitsMap.get(rightIndex); + if (cost == null) + return Double.POSITIVE_INFINITY; + return cost.doubleValue(); + } + +} diff --git a/marytts-unitselection/src/main/java/marytts/unitselection/select/SelectedUnit.java b/marytts-unitselection/src/main/java/marytts/unitselection/select/SelectedUnit.java new file mode 100644 index 0000000000..c6aa7891bb --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/unitselection/select/SelectedUnit.java @@ -0,0 +1,77 @@ +/** + * Copyright 2006 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.unitselection.select; + +import marytts.unitselection.data.Unit; +import marytts.features.Target; + +/** + * A unit selected from Viterbi + * + * @author Marc Schröder + * + */ +public class SelectedUnit { + protected Unit unit; + protected Target target; + protected Object concatenationData; + protected double[] audio; + + public SelectedUnit(Unit unit, Target target) { + this.unit = unit; + this.target = target; + this.audio = null; + } + + public Unit getUnit() { + return unit; + } + + public Target getTarget() { + return target; + } + + /** + * Remember data about this selected unit which is relevant for unit concatenation. What type of data is saved here depends on + * the UnitConcatenator used. + * + * @param concatenationData + */ + public void setConcatenationData(Object concatenationData) { + this.concatenationData = concatenationData; + } + + public Object getConcatenationData() { + return concatenationData; + } + + public void setAudio(double[] audio) { + this.audio = audio; + } + + public double[] getAudio() { + return audio; + } + + public String toString() { + return "Target: " + target.toString() + " Unit: " + unit.toString() + " target duration " + + target.getTargetDurationInSeconds(); + } +} diff --git a/marytts-unitselection/src/main/java/marytts/unitselection/select/StatisticalCostFunction.java b/marytts-unitselection/src/main/java/marytts/unitselection/select/StatisticalCostFunction.java new file mode 100644 index 0000000000..a322568954 --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/unitselection/select/StatisticalCostFunction.java @@ -0,0 +1,44 @@ +/** + * Copyright 2006 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.unitselection.select; + +import marytts.exceptions.MaryConfigurationException; +import marytts.unitselection.data.Unit; + +/** + * A statistical cost function + * + * @author Sathish Pammi + * + */ +public interface StatisticalCostFunction { + + public double cost(Unit u1, Unit u2); + + /** + * Initialise this scost cost function by reading the appropriate settings from the MaryProperties using the given + * configPrefix. + * + * @param configPrefix + * the prefix for the (voice-specific) config entries to use when looking up files to load. + */ + public void init(String configPrefix) throws MaryConfigurationException; + +} diff --git a/marytts-unitselection/src/main/java/marytts/unitselection/select/StatisticalModelCost.java b/marytts-unitselection/src/main/java/marytts/unitselection/select/StatisticalModelCost.java new file mode 100644 index 0000000000..069a533c99 --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/unitselection/select/StatisticalModelCost.java @@ -0,0 +1,143 @@ +/** + * Copyright 2006 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.unitselection.select; + +import marytts.exceptions.MaryConfigurationException; +import marytts.server.MaryProperties; +import marytts.unitselection.data.DiphoneUnit; +import marytts.unitselection.data.SCostFileReader; +import marytts.unitselection.data.Unit; +import marytts.util.data.MaryHeader; + +/** + * StatisticalModelCost for a given unit + * + * @author sathish pammi + * + */ +public class StatisticalModelCost implements StatisticalCostFunction { + + protected SCostFileReader sCostReader; + protected float sCostWeight; + + /****************/ + /* DATA FIELDS */ + /****************/ + private MaryHeader hdr = null; + + /****************/ + /* CONSTRUCTORS */ + /****************/ + + /** + * Empty constructor; when using this, call init() separately to initialise this class. + * + * @see #init(String) + */ + public StatisticalModelCost() { + } + + /** + * Initialise this scost function by reading the appropriate settings from the MaryProperties using the given configPrefix. + * + * @param configPrefix + * the prefix for the (voice-specific) config entries to use when looking up files to load. + */ + public void init(String configPrefix) throws MaryConfigurationException { + try { + String sCostFileName = MaryProperties.needFilename(configPrefix + ".sCostFile"); + sCostWeight = Float.parseFloat(MaryProperties.getProperty(configPrefix + ".sCostWeight", "1.0")); + sCostReader = new SCostFileReader(sCostFileName); + } catch (Exception e) { + throw new MaryConfigurationException("Cannot initialise scost model", e); + } + } + + /*****************/ + /* ACCESSORS */ + /*****************/ + + /** + * Get the number of units. + */ + public int getNumberOfUnits() { + return (sCostReader.getNumberOfUnits()); + } + + /*****************/ + /* MISC METHODS */ + /*****************/ + + public double cost(Unit u1, Unit u2) { + + // Units of length 0 cannot be joined: + if (u1.duration == 0 || u2.duration == 0) + return Double.POSITIVE_INFINITY; + // In the case of diphones, replace them with the relevant part: + if (u1 instanceof DiphoneUnit) { + u1 = ((DiphoneUnit) u1).right; + } + if (u2 instanceof DiphoneUnit) { + u2 = ((DiphoneUnit) u2).left; + } + + /** + * TODO uncomment below line to make ---> cost of successive units = 0 + */ + // if (u1.getIndex()+1 == u2.getIndex()) return 0; + + double sCost1 = sCostReader.getSCost(u1.index); + double sCost2 = sCostReader.getSCost(u2.index); + + return ((sCost1 + sCost2) / 2.0); + } + + /** + * Cost function for a given units if consecutive == true, and if they are consecutive units make cost = 0 + * + * @param u1 + * @param u2 + * @param consecutive + * @return + */ + public double cost(Unit u1, Unit u2, boolean consecutive) { + + // Units of length 0 cannot be joined: + if (u1.duration == 0 || u2.duration == 0) + return Double.POSITIVE_INFINITY; + // In the case of diphones, replace them with the relevant part: + if (u1 instanceof DiphoneUnit) { + u1 = ((DiphoneUnit) u1).right; + } + if (u2 instanceof DiphoneUnit) { + u2 = ((DiphoneUnit) u2).left; + } + + // if consecutive == true, and if they are consecutive units make cost = 0 + if (consecutive && (u1.index + 1 == u2.index)) + return 0; + + double sCost1 = sCostReader.getSCost(u1.index); + double sCost2 = sCostReader.getSCost(u2.index); + + return ((sCost1 + sCost2) / 2.0); + } + +} diff --git a/marytts-unitselection/src/main/java/marytts/unitselection/select/TargetCostFunction.java b/marytts-unitselection/src/main/java/marytts/unitselection/select/TargetCostFunction.java new file mode 100644 index 0000000000..b8a5f1cbeb --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/unitselection/select/TargetCostFunction.java @@ -0,0 +1,127 @@ +/** + * Copyright 2006 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.unitselection.select; + +import java.io.IOException; +import java.io.InputStream; + +import marytts.exceptions.MaryConfigurationException; +import marytts.features.FeatureDefinition; +import marytts.features.FeatureProcessorManager; +import marytts.features.FeatureVector; +import marytts.features.Target; +import marytts.unitselection.data.FeatureFileReader; +import marytts.unitselection.data.Unit; + +/** + * A target cost function for evaluating the goodness-of-fit of a given unit for a given target. + * + * @author Marc Schröder + * + */ +public interface TargetCostFunction { + /** + * Initialise the data needed to do a target cost computation. + * + * @param featureFileName + * name of a file containing the unit features + * @param weightsStream + * an optional weights file -- if non-null, contains feature weights that override the ones present in the feature + * file. + * @param featProc + * a feature processor manager which can provide feature processors to compute the features for a target at run + * time + * @throws IOException + * @throws MaryConfigurationException + * if a configuration problem is detected while loading the data + */ + public void load(String featureFileName, InputStream weightsStream, FeatureProcessorManager featProc) throws IOException, + MaryConfigurationException; + + /** + * Initialise the data needed to do a target cost computation. + * + * @param featureFileReader + * a reader for the file containing the unit features + * @param weightsFile + * an optional weights file -- if non-null, contains feature weights that override the ones present in the feature + * file. + * @param featProc + * a feature processor manager which can provide feature processors to compute the features for a target at run + * time + * @throws IOException + */ + public void load(FeatureFileReader featureFileReader, InputStream weightsStream, FeatureProcessorManager featProc) + throws IOException; + + /** + * Compute the goodness-of-fit of a given unit for a given target. + * + * @param target + * @param unit + * @return a non-negative number; smaller values mean better fit, i.e. smaller cost. + */ + public double cost(Target target, Unit unit); + + /** + * Compute the features for a given target, and store them in the target. + * + * @param target + * the target for which to compute the features + * @see Target#getFeatureVector() + */ + public void computeTargetFeatures(Target target); + + /** + * Provide access to the Feature Definition used. + * + * @return the feature definition object. + */ + public FeatureDefinition getFeatureDefinition(); + + /** + * Get the string representation of the feature value associated with the given unit + * + * @param unit + * the unit whose feature value is requested + * @param featureName + * name of the feature requested + * @return a string representation of the feature value + * @throws IllegalArgumentException + * if featureName is not a known feature + */ + public String getFeature(Unit unit, String featureName); + + /** + * Get the target cost feature vector for the given unit. + * + * @param unit + * @return + */ + public FeatureVector getFeatureVector(Unit unit); + + /** + * Get all feature vectors. This is useful for more efficient access. + * + * @return the full array of feature vectors, or null if this method is not supported. + */ + public FeatureVector[] getFeatureVectors(); + +} diff --git a/marytts-unitselection/src/main/java/marytts/unitselection/select/UnitSelector.java b/marytts-unitselection/src/main/java/marytts/unitselection/select/UnitSelector.java new file mode 100644 index 0000000000..24b2d2bade --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/unitselection/select/UnitSelector.java @@ -0,0 +1,159 @@ +/** + * Copyright 2006 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.unitselection.select; + +import java.util.ArrayList; +import java.util.List; + +import marytts.datatypes.MaryXML; +import marytts.exceptions.SynthesisException; +import marytts.features.Target; +import marytts.unitselection.data.UnitDatabase; +import marytts.unitselection.select.viterbi.Viterbi; +import marytts.util.MaryUtils; + +import org.apache.log4j.Logger; +import org.w3c.dom.Element; +import org.w3c.dom.NodeList; + +/** + * Selects the units for an utterance + * + * @author Marc Schröder + * + */ +public class UnitSelector { + protected UnitDatabase database; + protected Logger logger; + protected float targetCostWeight; + protected float sCostWeight = -1; + protected int beamSize; + + /** + * Initialise the unit selector. Need to call load() separately. + * + * @see #load(UnitDatabase) + */ + public UnitSelector() throws Exception { + logger = MaryUtils.getLogger(this.getClass()); + } + + public void load(UnitDatabase unitDatabase, float targetCostWeight, int beamSize) { + this.database = unitDatabase; + this.targetCostWeight = targetCostWeight; + this.beamSize = beamSize; + } + + public void load(UnitDatabase unitDatabase, float targetCostWeight, float sCostWeight, int beamSize) { + this.database = unitDatabase; + this.targetCostWeight = targetCostWeight; + this.sCostWeight = sCostWeight; + this.beamSize = beamSize; + } + + /** + * Select the units for the targets in the given list of tokens and boundaries. Collect them in a list and return it. + * + * @param tokensAndBoundaries + * the token and boundary MaryXML elements representing an utterance. + * @param voice + * the voice with which to synthesize + * @param db + * the database of the voice + * @param unitNamer + * a unitNamer + * @return a list of SelectedUnit objects + * @throws IllegalStateException + * if no path for generating the target utterance could be found + */ + public List selectUnits(List tokensAndBoundaries, marytts.modules.synthesis.Voice voice) + throws SynthesisException { + long time = System.currentTimeMillis(); + + List segmentsAndBoundaries = new ArrayList(); + for (Element tOrB : tokensAndBoundaries) { + if (tOrB.getTagName().equals(MaryXML.BOUNDARY)) { + segmentsAndBoundaries.add(tOrB); + } else { + assert tOrB.getTagName().equals(MaryXML.TOKEN) : "Expected token, got " + tOrB.getTagName(); + NodeList segs = tOrB.getElementsByTagName(MaryXML.PHONE); + for (int i = 0, max = segs.getLength(); i < max; i++) { + segmentsAndBoundaries.add((Element) segs.item(i)); + } + } + } + + List targets = createTargets(segmentsAndBoundaries); + // compute target features for each target in the chain + TargetCostFunction tcf = database.getTargetCostFunction(); + for (Target target : targets) { + tcf.computeTargetFeatures(target); + } + + Viterbi viterbi; + // Select the best candidates using Viterbi and the join cost function. + if (sCostWeight < 0) { + viterbi = new Viterbi(targets, database, targetCostWeight, beamSize); + } else { + viterbi = new Viterbi(targets, database, targetCostWeight, sCostWeight, beamSize); + } + + viterbi.apply(); + List selectedUnits = viterbi.getSelectedUnits(); + // If you can not associate the candidate units in the best path + // with the items in the segment relation, there is no best path + if (selectedUnits == null) { + throw new IllegalStateException("Viterbi: can't find path"); + } + long newtime = System.currentTimeMillis() - time; + logger.debug("Selection took " + newtime + " milliseconds"); + return selectedUnits; + } + + /** + * Create the list of targets from the XML elements to synthesize. + * + * @param segmentsAndBoundaries + * a list of MaryXML phone and boundary elements + * @return a list of Target objects + */ + protected List createTargets(List segmentsAndBoundaries) { + List targets = new ArrayList(); + for (Element sOrB : segmentsAndBoundaries) { + String phone = getPhoneSymbol(sOrB); + targets.add(new Target(phone, sOrB)); + } + return targets; + } + + public static String getPhoneSymbol(Element segmentOrBoundary) { + String phone; + if (segmentOrBoundary.getTagName().equals(MaryXML.PHONE)) { + phone = segmentOrBoundary.getAttribute("p"); + } else { + assert segmentOrBoundary.getTagName().equals(MaryXML.BOUNDARY) : "Expected boundary element, but got " + + segmentOrBoundary.getTagName(); + // TODO: how can we know the silence symbol here? + phone = "_"; + } + return phone; + } + +} diff --git a/marytts-unitselection/src/main/java/marytts/unitselection/select/viterbi/Viterbi.java b/marytts-unitselection/src/main/java/marytts/unitselection/select/viterbi/Viterbi.java new file mode 100644 index 0000000000..7cb18f44ad --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/unitselection/select/viterbi/Viterbi.java @@ -0,0 +1,532 @@ +/** + * Portions Copyright 2006 DFKI GmbH. + * Portions Copyright 2001 Sun Microsystems, Inc. + * Portions Copyright 1999-2001 Language Technologies Institute, + * Carnegie Mellon University. + * All Rights Reserved. Use is subject to license terms. + * + * Permission is hereby granted, free of charge, to use and distribute + * this software and its documentation without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of this work, and to + * permit persons to whom this work is furnished to do so, subject to + * the following conditions: + * + * 1. The code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * 2. Any modifications must be clearly marked as such. + * 3. Original authors' names are not deleted. + * 4. The authors' names are not used to endorse or promote products + * derived from this software without specific prior written + * permission. + * + * DFKI GMBH AND THE CONTRIBUTORS TO THIS WORK DISCLAIM ALL WARRANTIES WITH + * REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL DFKI GMBH NOR THE + * CONTRIBUTORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL + * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR + * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS + * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF + * THIS SOFTWARE. + */ +package marytts.unitselection.select.viterbi; + +import java.io.PrintWriter; +import java.io.StringWriter; +import java.text.DecimalFormat; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +import marytts.exceptions.SynthesisException; +import marytts.unitselection.data.DiphoneUnit; +import marytts.unitselection.data.Unit; +import marytts.unitselection.data.UnitDatabase; +import marytts.unitselection.select.JoinCostFunction; +import marytts.unitselection.select.SelectedUnit; +import marytts.unitselection.select.StatisticalCostFunction; +import marytts.unitselection.select.TargetCostFunction; +import marytts.features.Target; +import marytts.features.DiphoneTarget; +import marytts.util.MaryUtils; + +import org.apache.log4j.Level; +import org.apache.log4j.Logger; + +/** + * Provides support for the Viterbi Algorithm. + * + * Implementation Notes + *

+ * For each candidate for the current unit, calculate the cost between it and the first candidate in the next unit. Save only the + * path that has the least cost. By default, if two candidates come from units that are adjacent in the database, the cost is 0 + * (i.e., they were spoken together, so they are a perfect match). + *

+ * + * Repeat the previous process for each candidate in the next unit, creating a list of least cost paths between the candidates + * between the current unit and the unit following it. + *

+ * + * Toss out all candidates in the current unit that are not included in a path. + *

+ * + * Move to the next unit and repeat the process. + */ +public class Viterbi { + // a general flag indicating which type of viterbi search + // to use: + // -1: unlimited search + // n>0: beam search, retain only the n best paths at each step. + protected int beamSize; + protected final float wTargetCosts; + protected final float wJoinCosts; + protected final float wSCosts; + + protected ViterbiPoint firstPoint = null; + protected ViterbiPoint lastPoint = null; + private UnitDatabase database; + protected TargetCostFunction targetCostFunction; + protected JoinCostFunction joinCostFunction; + protected StatisticalCostFunction sCostFunction; + protected Logger logger; + // for debugging, try to get an idea of the average effect of join vs. target costs: + protected double cumulJoinCosts; + protected int nJoinCosts; + protected double cumulTargetCosts; + protected int nTargetCosts; + + // Keep track of average costs for each voice: map UnitDatabase->DebugStats + private static Map debugStats = new HashMap(); + + /** + * Creates a Viterbi class to process the given utterance. A queue of ViterbiPoints corresponding to the Items in the Relation + * segs is built up. + * + */ + public Viterbi(List targets, UnitDatabase database, float wTargetCosts, int beamSize) { + this.database = database; + this.targetCostFunction = database.getTargetCostFunction(); + this.joinCostFunction = database.getJoinCostFunction(); + this.sCostFunction = database.getSCostFunction(); + this.logger = MaryUtils.getLogger("Viterbi"); + this.wTargetCosts = wTargetCosts; + wJoinCosts = 1 - wTargetCosts; + wSCosts = 0; + this.beamSize = beamSize; + this.cumulJoinCosts = 0; + this.nJoinCosts = 0; + this.cumulTargetCosts = 0; + this.nTargetCosts = 0; + ViterbiPoint last = null; + // for each segment, build a ViterbiPoint + for (Target target : targets) { + ViterbiPoint nextPoint = new ViterbiPoint(target); + + if (last != null) { // continue to build up the queue + last.setNext(nextPoint); + } else { // firstPoint is the start of the queue + firstPoint = nextPoint; + // dummy start path: + firstPoint.getPaths().add(new ViterbiPath(null, null, 0)); + } + last = nextPoint; + } + // And add one point where the paths from the last candidate can end: + lastPoint = new ViterbiPoint(null); + last.setNext(lastPoint); + if (beamSize == 0) { + throw new IllegalStateException("General beam search not implemented"); + } + } + + /** + * Creates a Viterbi class to process the given utterance. A queue of ViterbiPoints corresponding to the Items in the Relation + * segs is built up. + * + */ + public Viterbi(List targets, UnitDatabase database, float wTargetCosts, float wSCosts, int beamSize) { + this.database = database; + this.targetCostFunction = database.getTargetCostFunction(); + this.joinCostFunction = database.getJoinCostFunction(); + this.sCostFunction = database.getSCostFunction(); + this.logger = MaryUtils.getLogger("Viterbi"); + this.wTargetCosts = wTargetCosts; + this.wSCosts = wSCosts; + wJoinCosts = 1 - (wTargetCosts + wSCosts); + this.beamSize = beamSize; + this.cumulJoinCosts = 0; + this.nJoinCosts = 0; + this.cumulTargetCosts = 0; + this.nTargetCosts = 0; + ViterbiPoint last = null; + // for each segment, build a ViterbiPoint + for (Target target : targets) { + ViterbiPoint nextPoint = new ViterbiPoint(target); + + if (last != null) { // continue to build up the queue + last.setNext(nextPoint); + } else { // firstPoint is the start of the queue + firstPoint = nextPoint; + // dummy start path: + firstPoint.getPaths().add(new ViterbiPath(null, null, 0)); + } + last = nextPoint; + } + // And add one point where the paths from the last candidate can end: + lastPoint = new ViterbiPoint(null); + last.setNext(lastPoint); + if (beamSize == 0) { + throw new IllegalStateException("General beam search not implemented"); + } + } + + /** + * Carry out a Viterbi search in for a prepared queue of ViterbiPoints. In a nutshell, each Point represents a target item (a + * target segment); for each target Point, a number of Candidate units in the voice database are determined; a Path structure + * is built up, based on local best transitions. Concretely, a Path consists of a (possibly empty) previous Path, a current + * Candidate, and a Score. This Score is a quality measure of the Path; it is calculated as the sum of the previous Path's + * score, the Candidate's score, and the Cost of joining the Candidate to the previous Path's Candidate. At each step, only + * one Path leading to each Candidate is retained, viz. the Path with the best Score. All that is left to do is to call + * result() to get the best-rated path from among the paths associated with the last Point, and to associate the resulting + * Candidates with the segment items they will realise. + * + * @throws SynthesisException + * if for any part of the target chain, no candidates can be found + */ + public void apply() throws SynthesisException { + logger.debug("Viterbi running with beam size " + beamSize); + // go through all but the last point + // (since last point has no item) + for (ViterbiPoint point = firstPoint; point.next != null; point = point.next) { + // The candidates for the current item: + // candidate selection is carried out by UnitSelector + Target target = point.target; + List candidates = database.getCandidates(target); + if (candidates.size() == 0) { + if (target instanceof DiphoneTarget) { + logger.debug("No diphone '" + target.getName() + "' -- will build from halfphones"); + DiphoneTarget dt = (DiphoneTarget) target; + // replace diphone viterbi point with two half-phone viterbi points + Target left = dt.left; + Target right = dt.right; + point.setTarget(left); + ViterbiPoint newP = new ViterbiPoint(right); + newP.next = point.next; + point.next = newP; + candidates = database.getCandidates(left); + if (candidates.size() == 0) + throw new SynthesisException("Cannot even find any halfphone unit for target " + left); + } else { + throw new SynthesisException("Cannot find any units for target " + target); + } + } + assert candidates.size() > 0; + + // absolutely critical since candidates is no longer a SortedSet: + Collections.sort(candidates); + + point.candidates = candidates; + assert beamSize != 0; // general beam search not implemented + + // Now go through all existing paths and all candidates + // for the current item; + // tentatively extend each existing path to each of + // the candidates, but only retain the best one + List paths = point.paths; + int nPaths = paths.size(); + if (beamSize != -1 && beamSize < nPaths) { + // beam search, look only at the best n paths: + nPaths = beamSize; + } + // for searchStrategy == -1, no beam -- look at all candidates. + int i = 0; + int iMax = nPaths; + for (ViterbiPath pp : paths) { + assert pp != null; + // We are at the very beginning of the search, + // or have a usable path to extend + candidates = point.candidates; + assert candidates != null; + int j = 0; + int jMax = beamSize; + // Go through the candidates as returned by the iterator of the sorted set, + // i.e. sorted according to increasing target cost. + for (ViterbiCandidate c : candidates) { + // For the candidate c, create a path extending the + // previous path pp to that candidate, taking into + // account the target and join costs: + ViterbiPath np = getPath(pp, c); + // Compare this path to the existing best path + // (if any) leading to candidate c; only retain + // the one with the better score. + addPath(point.next, np); + if (++j == jMax) + break; + } + if (++i == iMax) + break; + } + } + } + + /** + * Add the new path to the state path if it is better than the current path. In this, state means the position of the + * candidate associated with this path in the candidate queue for the corresponding segment item. In other words, this method + * uses newPath as the one path leading to the candidate newPath.candidate, if it has a better score than the previously best + * path leading to that candidate. + * + * @param point + * where the path is added + * @param newPath + * the path to add if its score is best + */ + void addPath(ViterbiPoint point, ViterbiPath newPath) { + // get the position of newPath's candidate + // in path array statePath of point + ViterbiCandidate candidate = newPath.candidate; + assert candidate != null; + ViterbiPath bestPathSoFar = candidate.bestPath; + List paths = point.getPaths(); + if (bestPathSoFar == null) { + // we don't have a path for the candidate yet, so this is best + paths.add(newPath); + candidate.setBestPath(newPath); + } else if (newPath.score < bestPathSoFar.score) { + // newPath is a better path for the candidate + paths.remove(bestPathSoFar); + paths.add(newPath); + candidate.setBestPath(newPath); + } + } + + /** + * Collect and return the best path, as a List of SelectedUnit objects. Note: This is a replacement for result(). + * + * @return the list of selected units, or null if no path could be found. + */ + public List getSelectedUnits() { + LinkedList selectedUnits = new LinkedList(); + if (firstPoint == null || firstPoint.getNext() == null) { + return selectedUnits; // null case + } + ViterbiPath best = findBestPath(); + if (best == null) { + // System.out.println("No best path found"); + return null; + } + for (ViterbiPath path = best; path != null; path = path.getPrevious()) { + if (path.candidate != null) { + Unit u = path.candidate.unit; + Target t = path.candidate.target; + if (u instanceof DiphoneUnit) { + assert t instanceof DiphoneTarget; + DiphoneUnit du = (DiphoneUnit) u; + DiphoneTarget dt = (DiphoneTarget) t; + selectedUnits.addFirst(new SelectedUnit(du.right, dt.right)); + selectedUnits.addFirst(new SelectedUnit(du.left, dt.left)); + } else { + selectedUnits.addFirst(new SelectedUnit(u, t)); + } + } + } + if (logger.getEffectiveLevel().equals(Level.DEBUG)) { + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw); + int prevIndex = -1; // index number of the previous unit + int[] lengthHistogram = new int[10]; + int length = 0; + int numUnits = selectedUnits.size(); + StringBuilder line = new StringBuilder(); + for (int i = 0; i < numUnits; i++) { + SelectedUnit u = (SelectedUnit) selectedUnits.get(i); + int index = u.getUnit().index; + if (prevIndex + 1 == index) { // adjacent units + length++; + } else { + if (lengthHistogram.length <= length) { + int[] dummy = new int[length + 1]; + System.arraycopy(lengthHistogram, 0, dummy, 0, lengthHistogram.length); + lengthHistogram = dummy; + } + lengthHistogram[length]++; + pw.print(line); + // Find filename from which the stretch that just finished + // stems: + if (i > 0) { + assert i >= length; + Unit firstUnitInStretch = ((SelectedUnit) selectedUnits.get(i - length)).getUnit(); + String origin = database.getFilenameAndTime(firstUnitInStretch); + // Print origin from column 80: + for (int col = line.length(); col < 80; col++) + pw.print(" "); + pw.print(origin); + } + pw.println(); + length = 1; + line.setLength(0); + } + line.append(database.getTargetCostFunction().getFeature(u.getUnit(), "phone") + "(" + u.getUnit().index + ")"); + prevIndex = index; + } + if (lengthHistogram.length <= length) { + int[] dummy = new int[length + 1]; + System.arraycopy(lengthHistogram, 0, dummy, 0, lengthHistogram.length); + lengthHistogram = dummy; + } + lengthHistogram[length]++; + pw.print(line); + // Find filename from which the stretch that just finished + // stems: + Unit firstUnitInStretch = ((SelectedUnit) selectedUnits.get(numUnits - length)).getUnit(); + String origin = database.getFilenameAndTime(firstUnitInStretch); + // Print origin from column 80: + for (int col = line.length(); col < 80; col++) + pw.print(" "); + pw.print(origin); + pw.println(); + logger.debug("Selected units:\n" + sw.toString()); + // Compute average length of stretches: + int total = 0; + int nStretches = 0; + for (int l = 1; l < lengthHistogram.length; l++) { + // lengthHistogram[0] will be 0 anyway + total += lengthHistogram[l] * l; + nStretches += lengthHistogram[l]; + } + float avgLength = total / (float) nStretches; + DecimalFormat df = new DecimalFormat("0.000"); + logger.debug("Avg. consecutive length: " + df.format(avgLength) + " units"); + // Cost of best path + double totalCost = best.score; + int elements = selectedUnits.size(); + double avgCostBestPath = totalCost / (elements - 1); + double avgTargetCost = cumulTargetCosts / nTargetCosts; + double avgJoinCost = cumulJoinCosts / nJoinCosts; + logger.debug("Avg. cost: best path " + df.format(avgCostBestPath) + ", avg. target " + df.format(avgTargetCost) + + ", join " + df.format(avgJoinCost) + " (n=" + nTargetCosts + ")"); + DebugStats stats = debugStats.get(database); + if (stats == null) { + stats = new DebugStats(); + debugStats.put(database, stats); + } + stats.n++; + // iterative computation of mean: + // m(n) = m(n-1) + (x(n) - m(n-1)) / n + stats.avgLength += (avgLength - stats.avgLength) / stats.n; + stats.avgCostBestPath += (avgCostBestPath - stats.avgCostBestPath) / stats.n; + stats.avgTargetCost += (avgTargetCost - stats.avgTargetCost) / stats.n; + stats.avgJoinCost += (avgJoinCost - stats.avgJoinCost) / stats.n; + logger.debug("Total average of " + stats.n + " utterances for this voice:"); + logger.debug("Avg. length: " + df.format(stats.avgLength) + ", avg. cost best path: " + + df.format(stats.avgCostBestPath) + ", avg. target cost: " + df.format(stats.avgTargetCost) + + ", avg. join cost: " + df.format(stats.avgJoinCost)); + + } + + return selectedUnits; + } + + /** + * Construct a new path element linking a previous path to the given candidate. The (penalty) score associated with the new + * path is calculated as the sum of the score of the old path plus the score of the candidate itself plus the join cost of + * appending the candidate to the nearest candidate in the given path. This join cost takes into account optimal coupling if + * the database has OPTIMAL_COUPLING set to 1. The join position is saved in the new path, as the features "unit_prev_move" + * and "unit_this_move". + * + * @param path + * the previous path, or null if this candidate starts a new path + * @param candiate + * the candidate to add to the path + * + * @return a new path, consisting of this candidate appended to the previous path, and with the cumulative (penalty) score + * calculated. + */ + private ViterbiPath getPath(ViterbiPath path, ViterbiCandidate candidate) { + double cost; + + Target candidateTarget = candidate.target; + Unit candidateUnit = candidate.unit; + + double joinCost; + double sCost = 0; + double targetCost; + // Target costs: + targetCost = candidate.targetCost; + + if (path == null || path.candidate == null) { + joinCost = 0; + } else { + // Join costs: + ViterbiCandidate prevCandidate = path.candidate; + Target prevTarget = prevCandidate.target; + Unit prevUnit = prevCandidate.unit; + joinCost = joinCostFunction.cost(prevTarget, prevUnit, candidateTarget, candidateUnit); + if (sCostFunction != null) + sCost = sCostFunction.cost(prevUnit, candidateUnit); + } + // Total cost is a weighted sum of join cost and target cost: + // cost = (1-r) * joinCost + r * targetCost, + // where r is given as the property "viterbi.wTargetCost" in a config file. + targetCost *= wTargetCosts; + joinCost *= wJoinCosts; + sCost *= wSCosts; + cost = joinCost + targetCost + sCost; + if (joinCost < Float.POSITIVE_INFINITY) + cumulJoinCosts += joinCost; + nJoinCosts++; + cumulTargetCosts += targetCost; + nTargetCosts++; + // logger.debug(candidateUnit+": target cost "+targetCost+", join cost "+joinCost); + + if (path != null) { + cost += path.score; + } + + return new ViterbiPath(candidate, path, cost); + } + + /** + * Find the best path. This requires apply() to have been run. For this best path, we set the pointers to the *next* path + * elements correctly. + * + * @return the best path, or null if no best path could be found. + */ + private ViterbiPath findBestPath() { + assert beamSize != 0; + // All paths end in lastPoint, and take into account + // previous path segment's scores. Therefore, it is + // sufficient to find the best path from among the + // paths for lastPoint. + List paths = lastPoint.getPaths(); + if (paths.isEmpty()) // no path, we failed + return null; + + // as paths is no longer a SortedSet, they must be explicitly sorted: + Collections.sort(paths); + ViterbiPath best = paths.get(0); + + // Set *next* pointers correctly: + ViterbiPath path = best; + double totalCost = best.score; + int elements = 0; + while (path != null) { + elements++; + ViterbiPath prev = path.previous; + if (prev != null) + prev.setNext(path); + path = prev; + } + return best; + } + + private class DebugStats { + int n; + double avgLength; + double avgCostBestPath; + double avgTargetCost; + double avgJoinCost; + } + +} diff --git a/marytts-unitselection/src/main/java/marytts/unitselection/select/viterbi/ViterbiCandidate.java b/marytts-unitselection/src/main/java/marytts/unitselection/select/viterbi/ViterbiCandidate.java new file mode 100644 index 0000000000..5a5c768a2d --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/unitselection/select/viterbi/ViterbiCandidate.java @@ -0,0 +1,142 @@ +/** + * Portions Copyright 2006 DFKI GmbH. + * Portions Copyright 2001 Sun Microsystems, Inc. + * Portions Copyright 1999-2001 Language Technologies Institute, + * Carnegie Mellon University. + * All Rights Reserved. Use is subject to license terms. + * + * Permission is hereby granted, free of charge, to use and distribute + * this software and its documentation without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of this work, and to + * permit persons to whom this work is furnished to do so, subject to + * the following conditions: + * + * 1. The code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * 2. Any modifications must be clearly marked as such. + * 3. Original authors' names are not deleted. + * 4. The authors' names are not used to endorse or promote products + * derived from this software without specific prior written + * permission. + * + * DFKI GMBH AND THE CONTRIBUTORS TO THIS WORK DISCLAIM ALL WARRANTIES WITH + * REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL DFKI GMBH NOR THE + * CONTRIBUTORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL + * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR + * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS + * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF + * THIS SOFTWARE. + */ +package marytts.unitselection.select.viterbi; + +import marytts.unitselection.data.Unit; +import marytts.features.Target; +import marytts.unitselection.select.TargetCostFunction; + +/** + * Represents a candidate for the Viterbi algorthm. Each candidate knows about its next candidate, i.e. they can form a queue. + */ +public class ViterbiCandidate implements Comparable { + final Target target; + final Unit unit; + final double targetCost; + ViterbiPath bestPath = null; + ViterbiCandidate next = null; + + public ViterbiCandidate(Target target, Unit unit, TargetCostFunction tcf) { + this.target = target; + this.unit = unit; + this.targetCost = tcf.cost(target, unit); + } + + /** + * Calculates and returns the target cost for this candidate + * + * @param tcf + * the target cost function + * @return the target cost + */ + public double getTargetCost() { + return targetCost; + } + + /** + * Gets the next candidate in the queue + * + * @return the next candidate + */ + public ViterbiCandidate getNext() { + return next; + } + + /** + * Sets the next candidate in the queue + * + * @param next + * the next candidate + */ + public void setNext(ViterbiCandidate next) { + this.next = next; + } + + /** + * Gets the target of this candidate + * + * @return the target + */ + public Target getTarget() { + return target; + } + + /** + * Gets the index of this + * + * @return the unit index + */ + public Unit getUnit() { + return unit; + } + + /** + * Sets the currently best path leading to this candidate. Each path leads to exactly one candidate; in the candidate, we only + * remember the best path leading to it. + * + * @param bestPath + */ + public void setBestPath(ViterbiPath bestPath) { + this.bestPath = bestPath; + } + + /** + * Gets the best path leading to this candidate + * + * @return the best path, or null + */ + public ViterbiPath getBestPath() { + return bestPath; + } + + /** + * Converts this object to a string. + * + * @return the string form of this object + */ + public String toString() { + return "ViterbiCandidate: target " + target + ", unit " + unit + + (bestPath != null ? ", best path score " + bestPath.score : ", no best path"); + } + + /** + * Compare two candidates so that the one with the smaller target cost is considered smaller. + */ + public int compareTo(ViterbiCandidate o) { + if (targetCost < o.targetCost) { + return -1; + } else if (targetCost > o.targetCost) { + return 1; + } + return 0; + } +} diff --git a/marytts-unitselection/src/main/java/marytts/unitselection/select/viterbi/ViterbiPath.java b/marytts-unitselection/src/main/java/marytts/unitselection/select/viterbi/ViterbiPath.java new file mode 100644 index 0000000000..32ebaa09ec --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/unitselection/select/viterbi/ViterbiPath.java @@ -0,0 +1,110 @@ +/** + * Portions Copyright 2006 DFKI GmbH. + * Portions Copyright 2001 Sun Microsystems, Inc. + * Portions Copyright 1999-2001 Language Technologies Institute, + * Carnegie Mellon University. + * All Rights Reserved. Use is subject to license terms. + * + * Permission is hereby granted, free of charge, to use and distribute + * this software and its documentation without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of this work, and to + * permit persons to whom this work is furnished to do so, subject to + * the following conditions: + * + * 1. The code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * 2. Any modifications must be clearly marked as such. + * 3. Original authors' names are not deleted. + * 4. The authors' names are not used to endorse or promote products + * derived from this software without specific prior written + * permission. + * + * DFKI GMBH AND THE CONTRIBUTORS TO THIS WORK DISCLAIM ALL WARRANTIES WITH + * REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL DFKI GMBH NOR THE + * CONTRIBUTORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL + * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR + * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS + * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF + * THIS SOFTWARE. + */ +package marytts.unitselection.select.viterbi; + +/** + * Describes a Viterbi path. + */ +public class ViterbiPath implements Comparable { + final double score; + final ViterbiCandidate candidate; + final ViterbiPath previous; + ViterbiPath next = null; + + public ViterbiPath(ViterbiCandidate candidate, ViterbiPath previousPath, double score) { + this.candidate = candidate; + this.previous = previousPath; + this.score = score; + } + + /** + * Get the score of this path + * + * @return the score + */ + public double getScore() { + return score; + } + + /** + * Get the candidate of this path. Each path leads to exactly one candidate. + * + * @return the candidate + */ + public ViterbiCandidate getCandidate() { + return candidate; + } + + /** + * Get the next path + * + * @return the next path + */ + public ViterbiPath getNext() { + return next; + } + + /** + * Set the next path + * + * @param next + * the next path + */ + public void setNext(ViterbiPath next) { + this.next = next; + } + + /** + * Get the previous path + * + * @return the previous path + */ + public ViterbiPath getPrevious() { + return previous; + } + + /** + * Converts this object to a string. + * + * @return the string form of this object + */ + public String toString() { + return "ViterbiPath score " + score + " leads to candidate unit " + candidate.getUnit(); + } + + /** + * Compare two viterbi paths such that the one with the lower score is considered smaller. + */ + public int compareTo(ViterbiPath o) { + return Double.compare(score, o.score); + } +} diff --git a/marytts-unitselection/src/main/java/marytts/unitselection/select/viterbi/ViterbiPoint.java b/marytts-unitselection/src/main/java/marytts/unitselection/select/viterbi/ViterbiPoint.java new file mode 100644 index 0000000000..1bfa94d0a4 --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/unitselection/select/viterbi/ViterbiPoint.java @@ -0,0 +1,129 @@ +/** + * Portions Copyright 2006 DFKI GmbH. + * Portions Copyright 2001 Sun Microsystems, Inc. + * Portions Copyright 1999-2001 Language Technologies Institute, + * Carnegie Mellon University. + * All Rights Reserved. Use is subject to license terms. + * + * Permission is hereby granted, free of charge, to use and distribute + * this software and its documentation without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of this work, and to + * permit persons to whom this work is furnished to do so, subject to + * the following conditions: + * + * 1. The code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * 2. Any modifications must be clearly marked as such. + * 3. Original authors' names are not deleted. + * 4. The authors' names are not used to endorse or promote products + * derived from this software without specific prior written + * permission. + * + * DFKI GMBH AND THE CONTRIBUTORS TO THIS WORK DISCLAIM ALL WARRANTIES WITH + * REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL DFKI GMBH NOR THE + * CONTRIBUTORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL + * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR + * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS + * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF + * THIS SOFTWARE. + */ +package marytts.unitselection.select.viterbi; + +import java.util.ArrayList; +import java.util.List; + +import marytts.features.Target; + +/** + * Represents a point in the Viterbi path. A point corresponds to an item, e.g. a Segment. Each ViterbiPoint knows about its next + * ViterbiPoint, i.e. they can form a queue. + */ +public class ViterbiPoint { + Target target = null; + List candidates = null; + List paths = new ArrayList(); + ViterbiPoint next = null; + + /** + * Creates a ViterbiPoint for the given target. + * + * @param target + * the target of interest + */ + public ViterbiPoint(Target target) { + this.target = target; + } + + /** + * Gets the target of this point + * + * @return the target + */ + public Target getTarget() { + return target; + } + + /** + * Sets the target of this point + * + * @param target + * the new target + */ + public void setTarget(Target target) { + this.target = target; + } + + /** + * Gets the candidates of this point + * + * @return the candidates + */ + public List getCandidates() { + return candidates; + } + + /** + * Sets the candidates of this point + * + * @param cands + * the candidates + */ + public void setCandidates(ArrayList candidates) { + this.candidates = candidates; + } + + /** + * Gets the sorted set containting the paths of the candidates of this point, sorted by score. getPaths().first() will return + * the path with the lowest score, i.e. the best path. + * + * @return a sorted set. + */ + public List getPaths() { + return paths; + } + + /** + * Gets the next point in the queue + * + * @return the next point + */ + public ViterbiPoint getNext() { + return next; + } + + /** + * Sets the next point in the queue + * + * @param next + * the next point + */ + public void setNext(ViterbiPoint next) { + this.next = next; + } + + public String toString() { + return "ViterbiPoint: target " + target + "; " + paths.size() + " paths"; + } +} diff --git a/marytts-unitselection/src/main/java/marytts/unitselection/weightingfunctions/WeightFunc.java b/marytts-unitselection/src/main/java/marytts/unitselection/weightingfunctions/WeightFunc.java new file mode 100644 index 0000000000..991027d64a --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/unitselection/weightingfunctions/WeightFunc.java @@ -0,0 +1,49 @@ +/** + * Portions Copyright 2006 DFKI GmbH. + * Portions Copyright 2001 Sun Microsystems, Inc. + * Portions Copyright 1999-2001 Language Technologies Institute, + * Carnegie Mellon University. + * All Rights Reserved. Use is subject to license terms. + * + * Permission is hereby granted, free of charge, to use and distribute + * this software and its documentation without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of this work, and to + * permit persons to whom this work is furnished to do so, subject to + * the following conditions: + * + * 1. The code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * 2. Any modifications must be clearly marked as such. + * 3. Original authors' names are not deleted. + * 4. The authors' names are not used to endorse or promote products + * derived from this software without specific prior written + * permission. + * + * DFKI GMBH AND THE CONTRIBUTORS TO THIS WORK DISCLAIM ALL WARRANTIES WITH + * REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL DFKI GMBH NOR THE + * CONTRIBUTORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL + * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR + * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS + * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF + * THIS SOFTWARE. + */ +package marytts.unitselection.weightingfunctions; + +/** + * A uniform interface for the weighting functions. + * + * @author sacha + * + */ +public interface WeightFunc { + /** Compute a weighted difference */ + public double cost(double a, double b); + + /** Possibly set optional parameters. */ + public void setParam(String val); + + /** Return a weighting function definition string. */ + public String whoAmI(); +} diff --git a/marytts-unitselection/src/main/java/marytts/unitselection/weightingfunctions/WeightFunctionManager.java b/marytts-unitselection/src/main/java/marytts/unitselection/weightingfunctions/WeightFunctionManager.java new file mode 100644 index 0000000000..ad7c767d9e --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/unitselection/weightingfunctions/WeightFunctionManager.java @@ -0,0 +1,99 @@ +/** + * Portions Copyright 2006 DFKI GmbH. + * Portions Copyright 2001 Sun Microsystems, Inc. + * Portions Copyright 1999-2001 Language Technologies Institute, + * Carnegie Mellon University. + * All Rights Reserved. Use is subject to license terms. + * + * Permission is hereby granted, free of charge, to use and distribute + * this software and its documentation without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of this work, and to + * permit persons to whom this work is furnished to do so, subject to + * the following conditions: + * + * 1. The code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * 2. Any modifications must be clearly marked as such. + * 3. Original authors' names are not deleted. + * 4. The authors' names are not used to endorse or promote products + * derived from this software without specific prior written + * permission. + * + * DFKI GMBH AND THE CONTRIBUTORS TO THIS WORK DISCLAIM ALL WARRANTIES WITH + * REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL DFKI GMBH NOR THE + * CONTRIBUTORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL + * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR + * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS + * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF + * THIS SOFTWARE. + */ + +package marytts.unitselection.weightingfunctions; + +import java.util.HashMap; +import java.util.Map; + +/** + * This class connects weighting function names with the actual instances of the weighting functions. + * + * @author sacha + * + */ +public class WeightFunctionManager { + + public static Map weightFuncMap; + + static { + weightFuncMap = new HashMap(); + weightFuncMap.put("linear", new WeightingFunction.linear()); + weightFuncMap.put("step", new WeightingFunction.step()); + } + + /** + * Dummy empty contructor. + */ + public WeightFunctionManager() { + } + + /** + * Accessor for the hash map mapping names to interface instances. + * + * @return a hash map. + */ + public static Map getWeightFunc() { + return weightFuncMap; + } + + /** + * Returns the weighting function from its name. + * + * @param funcName + * The name of the weighting function. + * @return an interface to a weighting function. + */ + public WeightFunc getWeightFunction(String funcName) { + /* Split the string in 2 parts: function name plus parameters */ + String[] strPart = funcName.split("\\s", 2); + WeightFunc wf = (WeightFunc) weightFuncMap.get(strPart[0]); + /* If the function asked for does not exist, inform the user and throw an exception. */ + if (wf == null) { + String[] known = (String[]) weightFuncMap.keySet().toArray(new String[0]); + String strKnown = known[0]; + for (int i = 1; i < known.length; i++) { + strKnown = strKnown + "; " + known[i]; + } + strKnown = strKnown + "."; + throw new RuntimeException("The weighting manager was asked for the unknown weighting function type [" + funcName + + "]. Known types are: " + strKnown); + } + /* If the function has a parameter, parse and set it */ + // TODO: This is not thread-safe! What if several threads call wf.setParam() with different values? + if (strPart.length > 1) + wf.setParam(strPart[1]); + /* Return the functionś interface */ + return (wf); + } + +} diff --git a/marytts-unitselection/src/main/java/marytts/unitselection/weightingfunctions/WeightingFunction.java b/marytts-unitselection/src/main/java/marytts/unitselection/weightingfunctions/WeightingFunction.java new file mode 100644 index 0000000000..d0b75996c1 --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/unitselection/weightingfunctions/WeightingFunction.java @@ -0,0 +1,119 @@ +/** + * Portions Copyright 2006 DFKI GmbH. + * Portions Copyright 2001 Sun Microsystems, Inc. + * Portions Copyright 1999-2001 Language Technologies Institute, + * Carnegie Mellon University. + * All Rights Reserved. Use is subject to license terms. + * + * Permission is hereby granted, free of charge, to use and distribute + * this software and its documentation without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of this work, and to + * permit persons to whom this work is furnished to do so, subject to + * the following conditions: + * + * 1. The code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * 2. Any modifications must be clearly marked as such. + * 3. Original authors' names are not deleted. + * 4. The authors' names are not used to endorse or promote products + * derived from this software without specific prior written + * permission. + * + * DFKI GMBH AND THE CONTRIBUTORS TO THIS WORK DISCLAIM ALL WARRANTIES WITH + * REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL DFKI GMBH NOR THE + * CONTRIBUTORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL + * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR + * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS + * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF + * THIS SOFTWARE. + */ +package marytts.unitselection.weightingfunctions; + +/** + * Defines the applicable weighting functions. + * + * @author sacha + * + */ +public class WeightingFunction { + + /** + * Linear weighting function: just computes the difference. + * + * @author sacha + * + */ + public static class linear implements WeightFunc { + /** + * Cost computation: a simple absolute value of a subtraction. + */ + public double cost(double a, double b) { + return (a > b ? (a - b) : (b - a)); + /* + * (Measuring which one is bigger replaces a costly Math.abs() operation.) + */ + } + + /** Optional argument setting. */ + public void setParam(String val) { + /* + * Does nothing and is never called in the linear case. + */ + } + + /** + * Returns the string definition for this weight function. + */ + public String whoAmI() { + return ("linear"); + } + } + + /** + * Step weighting function: saturates above a given percentage of the input values. + * + * @author sacha + * + */ + public static class step implements WeightFunc { + /** A percentage above which the function saturates to 0. */ + private double stepVal; + + /** + * Cost computation: the absolute value of a subtraction, with application of a saturation if the difference value reaches + * a certain percentage of the mean value of the arguments. + */ + public double cost(double a, double b) { + + double res = (a > b ? (a - b) : (b - a)); + /* + * (Measuring which one is bigger replaces a costly Math.abs() operation.) + */ + + double dev = res / ((a + b) / 2.0); + if (dev < stepVal) + return (res); + else + return (0.0); + } + + /** + * Optional argument setting. + * + * Syntax for the step function's parameter: the first number before the % sign is interpreted as a percentage for the + * step value. + * + * */ + public void setParam(String val) { + stepVal = Double.parseDouble(val.substring(0, val.indexOf("%"))) / 100.0; + } + + /** String definition of the function. */ + public String whoAmI() { + return ("step " + Double.toString(100.0 * stepVal) + "%"); + } + } + +} From bc75cdb4f0c6acf507ee71ff661d804baa28ec5e Mon Sep 17 00:00:00 2001 From: Tristan Hamilton Date: Thu, 5 Feb 2015 14:27:32 +0100 Subject: [PATCH 07/31] remove trove4j dependancy from marytts-runtime pom, add to marytts-unitselection pom --- marytts-runtime/pom.xml | 5 ----- marytts-unitselection/pom.xml | 10 ++++++++-- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/marytts-runtime/pom.xml b/marytts-runtime/pom.xml index fc21e003ff..987861c2a0 100644 --- a/marytts-runtime/pom.xml +++ b/marytts-runtime/pom.xml @@ -74,11 +74,6 @@ guava - - net.sf.trove4j - trove4j - - org.apache.httpcomponents httpcore diff --git a/marytts-unitselection/pom.xml b/marytts-unitselection/pom.xml index 00b112dcbd..0f5b586a6f 100644 --- a/marytts-unitselection/pom.xml +++ b/marytts-unitselection/pom.xml @@ -8,11 +8,17 @@ marytts-unitselection ${project.artifactId} + - de.dfki.mary + ${project.groupId} marytts-runtime - 5.2-SNAPSHOT + ${project.version} + + + net.sf.trove4j + trove4j + \ No newline at end of file From 76d492a2150ef0704726a6f0d1b75329b0b94b9a Mon Sep 17 00:00:00 2001 From: Tristan Hamilton Date: Thu, 5 Feb 2015 14:32:52 +0100 Subject: [PATCH 08/31] clean up references to Target and its subclasses --- marytts-runtime/src/main/java/marytts/cart/CART.java | 2 +- marytts-runtime/src/main/java/marytts/cart/DirectedGraph.java | 2 +- .../src/main/java/marytts/cart/StringPredictionTree.java | 2 +- .../java/marytts/features/ByteValuedFeatureProcessor.java | 1 - .../java/marytts/features/ContinuousFeatureProcessor.java | 1 - .../java/marytts/features/MaryGenericFeatureProcessors.java | 3 --- .../java/marytts/features/MaryLanguageFeatureProcessors.java | 2 -- .../java/marytts/features/ShortValuedFeatureProcessor.java | 1 - .../src/main/java/marytts/features/TargetFeatureComputer.java | 1 - .../src/main/java/marytts/machinelearning/SoP.java | 2 +- .../src/main/java/marytts/modules/CARTDurationModeller.java | 3 +-- .../src/main/java/marytts/modules/CARTF0Modeller.java | 2 +- marytts-runtime/src/main/java/marytts/modules/HTSEngine.java | 2 +- .../java/marytts/modules/HalfPhoneTargetFeatureLister.java | 4 ++-- .../src/main/java/marytts/modules/PolynomialF0Modeller.java | 2 +- .../src/main/java/marytts/modules/PronunciationModel.java | 2 +- .../src/main/java/marytts/modules/SoPDurationModeller.java | 2 +- .../src/main/java/marytts/modules/SoPF0Modeller.java | 2 +- .../src/main/java/marytts/modules/TargetFeatureLister.java | 2 +- .../src/main/java/marytts/modules/acoustic/BoundaryModel.java | 2 +- .../src/main/java/marytts/modules/acoustic/CARTModel.java | 2 +- .../src/main/java/marytts/modules/acoustic/HMMModel.java | 2 +- .../src/main/java/marytts/modules/acoustic/Model.java | 2 +- .../src/main/java/marytts/modules/acoustic/SoPModel.java | 2 +- .../main/java/marytts/modules/synthesis/HMMSynthesizer.java | 2 +- 25 files changed, 20 insertions(+), 30 deletions(-) diff --git a/marytts-runtime/src/main/java/marytts/cart/CART.java b/marytts-runtime/src/main/java/marytts/cart/CART.java index 2ab751a423..ac65c1ad85 100644 --- a/marytts-runtime/src/main/java/marytts/cart/CART.java +++ b/marytts-runtime/src/main/java/marytts/cart/CART.java @@ -35,7 +35,7 @@ import marytts.features.FeatureDefinition; import marytts.features.FeatureVector; -import marytts.unitselection.select.Target; +import marytts.features.Target; /** * A tree is a specific kind of directed graph in which each node can have only a single parent node. It consists exclusively of diff --git a/marytts-runtime/src/main/java/marytts/cart/DirectedGraph.java b/marytts-runtime/src/main/java/marytts/cart/DirectedGraph.java index e96e22746b..3d3e1db64e 100644 --- a/marytts-runtime/src/main/java/marytts/cart/DirectedGraph.java +++ b/marytts-runtime/src/main/java/marytts/cart/DirectedGraph.java @@ -25,7 +25,7 @@ import marytts.features.FeatureDefinition; import marytts.features.FeatureVector; -import marytts.unitselection.select.Target; +import marytts.features.Target; import marytts.util.MaryUtils; import org.apache.log4j.Logger; diff --git a/marytts-runtime/src/main/java/marytts/cart/StringPredictionTree.java b/marytts-runtime/src/main/java/marytts/cart/StringPredictionTree.java index bb74e67244..8f9c7c902a 100644 --- a/marytts-runtime/src/main/java/marytts/cart/StringPredictionTree.java +++ b/marytts-runtime/src/main/java/marytts/cart/StringPredictionTree.java @@ -28,7 +28,7 @@ import marytts.cart.io.WagonCARTReader; import marytts.features.FeatureDefinition; import marytts.features.FeatureVector; -import marytts.unitselection.select.Target; +import marytts.features.Target; //import com.sun.tools.javac.code.Attribute.Array; diff --git a/marytts-runtime/src/main/java/marytts/features/ByteValuedFeatureProcessor.java b/marytts-runtime/src/main/java/marytts/features/ByteValuedFeatureProcessor.java index 71c007946a..68c064e420 100644 --- a/marytts-runtime/src/main/java/marytts/features/ByteValuedFeatureProcessor.java +++ b/marytts-runtime/src/main/java/marytts/features/ByteValuedFeatureProcessor.java @@ -31,7 +31,6 @@ */ package marytts.features; -import marytts.unitselection.select.Target; /** * Performs a specific type of processing on an item and returns an object. diff --git a/marytts-runtime/src/main/java/marytts/features/ContinuousFeatureProcessor.java b/marytts-runtime/src/main/java/marytts/features/ContinuousFeatureProcessor.java index 5d6a851379..afdb4667d6 100644 --- a/marytts-runtime/src/main/java/marytts/features/ContinuousFeatureProcessor.java +++ b/marytts-runtime/src/main/java/marytts/features/ContinuousFeatureProcessor.java @@ -31,7 +31,6 @@ */ package marytts.features; -import marytts.unitselection.select.Target; /** * Performs a specific type of processing on an item and returns an object. diff --git a/marytts-runtime/src/main/java/marytts/features/MaryGenericFeatureProcessors.java b/marytts-runtime/src/main/java/marytts/features/MaryGenericFeatureProcessors.java index 0436d70a83..ba8dc4be0b 100644 --- a/marytts-runtime/src/main/java/marytts/features/MaryGenericFeatureProcessors.java +++ b/marytts-runtime/src/main/java/marytts/features/MaryGenericFeatureProcessors.java @@ -39,9 +39,6 @@ import java.util.StringTokenizer; import marytts.datatypes.MaryXML; -import marytts.unitselection.select.DiphoneTarget; -import marytts.unitselection.select.HalfPhoneTarget; -import marytts.unitselection.select.Target; import marytts.util.dom.MaryDomUtils; import marytts.util.string.ByteStringTranslator; diff --git a/marytts-runtime/src/main/java/marytts/features/MaryLanguageFeatureProcessors.java b/marytts-runtime/src/main/java/marytts/features/MaryLanguageFeatureProcessors.java index 18548f4637..5813f90c61 100644 --- a/marytts-runtime/src/main/java/marytts/features/MaryLanguageFeatureProcessors.java +++ b/marytts-runtime/src/main/java/marytts/features/MaryLanguageFeatureProcessors.java @@ -37,8 +37,6 @@ import marytts.datatypes.MaryXML; import marytts.fst.FSTLookup; import marytts.modules.phonemiser.AllophoneSet; -import marytts.unitselection.select.HalfPhoneTarget; -import marytts.unitselection.select.Target; import marytts.util.dom.MaryDomUtils; import marytts.util.string.ByteStringTranslator; diff --git a/marytts-runtime/src/main/java/marytts/features/ShortValuedFeatureProcessor.java b/marytts-runtime/src/main/java/marytts/features/ShortValuedFeatureProcessor.java index d045340ebd..bd36a1048b 100644 --- a/marytts-runtime/src/main/java/marytts/features/ShortValuedFeatureProcessor.java +++ b/marytts-runtime/src/main/java/marytts/features/ShortValuedFeatureProcessor.java @@ -31,7 +31,6 @@ */ package marytts.features; -import marytts.unitselection.select.Target; /** * Performs a specific type of processing on an item and returns an object. diff --git a/marytts-runtime/src/main/java/marytts/features/TargetFeatureComputer.java b/marytts-runtime/src/main/java/marytts/features/TargetFeatureComputer.java index b7ef7a5377..078ab325e5 100644 --- a/marytts-runtime/src/main/java/marytts/features/TargetFeatureComputer.java +++ b/marytts-runtime/src/main/java/marytts/features/TargetFeatureComputer.java @@ -26,7 +26,6 @@ import java.util.List; import java.util.StringTokenizer; -import marytts.unitselection.select.Target; /** * Compute a given set of features for a Target. diff --git a/marytts-runtime/src/main/java/marytts/machinelearning/SoP.java b/marytts-runtime/src/main/java/marytts/machinelearning/SoP.java index f4c0f81203..6a93149a76 100644 --- a/marytts-runtime/src/main/java/marytts/machinelearning/SoP.java +++ b/marytts-runtime/src/main/java/marytts/machinelearning/SoP.java @@ -27,7 +27,7 @@ import java.util.Scanner; import marytts.features.FeatureDefinition; -import marytts.unitselection.select.Target; +import marytts.features.Target; /** * Contains the coefficients and factors of an equation of the form: if interceptTterm = TRUE solution = coeffs[0] + diff --git a/marytts-runtime/src/main/java/marytts/modules/CARTDurationModeller.java b/marytts-runtime/src/main/java/marytts/modules/CARTDurationModeller.java index 89f811ed18..9e9b2abff9 100644 --- a/marytts-runtime/src/main/java/marytts/modules/CARTDurationModeller.java +++ b/marytts-runtime/src/main/java/marytts/modules/CARTDurationModeller.java @@ -35,10 +35,9 @@ import marytts.features.FeatureProcessorManager; import marytts.features.FeatureRegistry; import marytts.features.TargetFeatureComputer; +import marytts.features.Target; import marytts.modules.synthesis.Voice; import marytts.server.MaryProperties; -import marytts.unitselection.select.Target; -import marytts.unitselection.select.UnitSelector; import marytts.util.MaryRuntimeUtils; import marytts.util.MaryUtils; import marytts.util.dom.MaryDomUtils; diff --git a/marytts-runtime/src/main/java/marytts/modules/CARTF0Modeller.java b/marytts-runtime/src/main/java/marytts/modules/CARTF0Modeller.java index a82ddf7fa1..91ca34f239 100644 --- a/marytts-runtime/src/main/java/marytts/modules/CARTF0Modeller.java +++ b/marytts-runtime/src/main/java/marytts/modules/CARTF0Modeller.java @@ -39,7 +39,7 @@ import marytts.modules.synthesis.Voice; import marytts.server.MaryProperties; import marytts.unitselection.UnitSelectionVoice; -import marytts.unitselection.select.Target; +import marytts.features.Target; import marytts.util.MaryRuntimeUtils; import marytts.util.MaryUtils; import marytts.util.dom.MaryDomUtils; diff --git a/marytts-runtime/src/main/java/marytts/modules/HTSEngine.java b/marytts-runtime/src/main/java/marytts/modules/HTSEngine.java index de2d7d0b4f..a35ca67529 100644 --- a/marytts-runtime/src/main/java/marytts/modules/HTSEngine.java +++ b/marytts-runtime/src/main/java/marytts/modules/HTSEngine.java @@ -93,7 +93,7 @@ import marytts.htsengine.HTSVocoder; import marytts.htsengine.HTSEngineTest.PhonemeDuration; import marytts.modules.synthesis.Voice; -import marytts.unitselection.select.Target; +import marytts.features.Target; import marytts.util.MaryUtils; import marytts.util.data.audio.AppendableSequenceAudioInputStream; import marytts.util.data.audio.AudioPlayer; diff --git a/marytts-runtime/src/main/java/marytts/modules/HalfPhoneTargetFeatureLister.java b/marytts-runtime/src/main/java/marytts/modules/HalfPhoneTargetFeatureLister.java index 41d0ed6b7f..515dcefff5 100644 --- a/marytts-runtime/src/main/java/marytts/modules/HalfPhoneTargetFeatureLister.java +++ b/marytts-runtime/src/main/java/marytts/modules/HalfPhoneTargetFeatureLister.java @@ -24,9 +24,9 @@ import marytts.datatypes.MaryDataType; import marytts.datatypes.MaryXML; -import marytts.unitselection.select.HalfPhoneTarget; -import marytts.unitselection.select.Target; import marytts.unitselection.select.UnitSelector; +import marytts.features.Target; +import marytts.features.HalfPhoneTarget; import marytts.util.dom.MaryDomUtils; import org.w3c.dom.Element; diff --git a/marytts-runtime/src/main/java/marytts/modules/PolynomialF0Modeller.java b/marytts-runtime/src/main/java/marytts/modules/PolynomialF0Modeller.java index c51aa9f206..cea5f6b912 100644 --- a/marytts-runtime/src/main/java/marytts/modules/PolynomialF0Modeller.java +++ b/marytts-runtime/src/main/java/marytts/modules/PolynomialF0Modeller.java @@ -34,7 +34,7 @@ import marytts.modules.synthesis.Voice; import marytts.server.MaryProperties; import marytts.unitselection.data.FeatureFileReader; -import marytts.unitselection.select.Target; +import marytts.features.Target; import marytts.util.MaryRuntimeUtils; import marytts.util.MaryUtils; import marytts.util.dom.MaryDomUtils; diff --git a/marytts-runtime/src/main/java/marytts/modules/PronunciationModel.java b/marytts-runtime/src/main/java/marytts/modules/PronunciationModel.java index e7eb2c6008..d788ab872e 100644 --- a/marytts-runtime/src/main/java/marytts/modules/PronunciationModel.java +++ b/marytts-runtime/src/main/java/marytts/modules/PronunciationModel.java @@ -41,7 +41,7 @@ import marytts.modules.phonemiser.Allophone; import marytts.modules.phonemiser.AllophoneSet; import marytts.server.MaryProperties; -import marytts.unitselection.select.Target; +import marytts.features.Target; import marytts.util.MaryRuntimeUtils; import marytts.util.dom.MaryDomUtils; diff --git a/marytts-runtime/src/main/java/marytts/modules/SoPDurationModeller.java b/marytts-runtime/src/main/java/marytts/modules/SoPDurationModeller.java index 5f9e8cf854..39123582a1 100644 --- a/marytts-runtime/src/main/java/marytts/modules/SoPDurationModeller.java +++ b/marytts-runtime/src/main/java/marytts/modules/SoPDurationModeller.java @@ -36,8 +36,8 @@ import marytts.modules.phonemiser.AllophoneSet; import marytts.modules.synthesis.Voice; import marytts.server.MaryProperties; -import marytts.unitselection.select.Target; import marytts.unitselection.select.UnitSelector; +import marytts.features.Target; import marytts.util.MaryRuntimeUtils; import marytts.util.MaryUtils; import marytts.util.dom.MaryDomUtils; diff --git a/marytts-runtime/src/main/java/marytts/modules/SoPF0Modeller.java b/marytts-runtime/src/main/java/marytts/modules/SoPF0Modeller.java index 570814c281..65e76fe5e3 100644 --- a/marytts-runtime/src/main/java/marytts/modules/SoPF0Modeller.java +++ b/marytts-runtime/src/main/java/marytts/modules/SoPF0Modeller.java @@ -38,7 +38,7 @@ import marytts.modules.synthesis.Voice; import marytts.server.MaryProperties; import marytts.unitselection.UnitSelectionVoice; -import marytts.unitselection.select.Target; +import marytts.features.Target; import marytts.util.MaryRuntimeUtils; import marytts.util.MaryUtils; import marytts.util.dom.MaryDomUtils; diff --git a/marytts-runtime/src/main/java/marytts/modules/TargetFeatureLister.java b/marytts-runtime/src/main/java/marytts/modules/TargetFeatureLister.java index 2479077384..5ba8b46ce5 100644 --- a/marytts-runtime/src/main/java/marytts/modules/TargetFeatureLister.java +++ b/marytts-runtime/src/main/java/marytts/modules/TargetFeatureLister.java @@ -30,8 +30,8 @@ import marytts.features.FeatureVector; import marytts.features.TargetFeatureComputer; import marytts.modules.synthesis.Voice; -import marytts.unitselection.select.Target; import marytts.unitselection.select.UnitSelector; +import marytts.features.Target; import marytts.util.dom.MaryDomUtils; import org.w3c.dom.Document; diff --git a/marytts-runtime/src/main/java/marytts/modules/acoustic/BoundaryModel.java b/marytts-runtime/src/main/java/marytts/modules/acoustic/BoundaryModel.java index 1dfe64dc25..6a77a6d7bc 100644 --- a/marytts-runtime/src/main/java/marytts/modules/acoustic/BoundaryModel.java +++ b/marytts-runtime/src/main/java/marytts/modules/acoustic/BoundaryModel.java @@ -24,7 +24,7 @@ import java.util.List; import marytts.features.FeatureProcessorManager; -import marytts.unitselection.select.Target; +import marytts.features.Target; import org.w3c.dom.Element; diff --git a/marytts-runtime/src/main/java/marytts/modules/acoustic/CARTModel.java b/marytts-runtime/src/main/java/marytts/modules/acoustic/CARTModel.java index ecd38320e2..70c6214442 100644 --- a/marytts-runtime/src/main/java/marytts/modules/acoustic/CARTModel.java +++ b/marytts-runtime/src/main/java/marytts/modules/acoustic/CARTModel.java @@ -28,7 +28,7 @@ import marytts.cart.io.DirectedGraphReader; import marytts.exceptions.MaryConfigurationException; import marytts.features.FeatureProcessorManager; -import marytts.unitselection.select.Target; +import marytts.features.Target; /** * Model for applying a CART to a list of Targets diff --git a/marytts-runtime/src/main/java/marytts/modules/acoustic/HMMModel.java b/marytts-runtime/src/main/java/marytts/modules/acoustic/HMMModel.java index 810ae15f55..f4c33cdaf3 100644 --- a/marytts-runtime/src/main/java/marytts/modules/acoustic/HMMModel.java +++ b/marytts-runtime/src/main/java/marytts/modules/acoustic/HMMModel.java @@ -36,7 +36,7 @@ import marytts.htsengine.HTSModel; import marytts.htsengine.HTSParameterGeneration; import marytts.htsengine.HTSUttModel; -import marytts.unitselection.select.Target; +import marytts.features.Target; import marytts.util.MaryUtils; import org.apache.log4j.Logger; diff --git a/marytts-runtime/src/main/java/marytts/modules/acoustic/Model.java b/marytts-runtime/src/main/java/marytts/modules/acoustic/Model.java index e575ad35ec..c972490306 100644 --- a/marytts-runtime/src/main/java/marytts/modules/acoustic/Model.java +++ b/marytts-runtime/src/main/java/marytts/modules/acoustic/Model.java @@ -31,8 +31,8 @@ import marytts.features.FeatureRegistry; import marytts.features.FeatureVector; import marytts.features.TargetFeatureComputer; -import marytts.unitselection.select.Target; import marytts.unitselection.select.UnitSelector; +import marytts.features.Target; import org.w3c.dom.Element; import org.w3c.dom.Node; diff --git a/marytts-runtime/src/main/java/marytts/modules/acoustic/SoPModel.java b/marytts-runtime/src/main/java/marytts/modules/acoustic/SoPModel.java index 840233a1d9..1017926142 100644 --- a/marytts-runtime/src/main/java/marytts/modules/acoustic/SoPModel.java +++ b/marytts-runtime/src/main/java/marytts/modules/acoustic/SoPModel.java @@ -34,7 +34,7 @@ import marytts.features.FeatureDefinition; import marytts.features.FeatureProcessorManager; import marytts.machinelearning.SoP; -import marytts.unitselection.select.Target; +import marytts.features.Target; import marytts.util.MaryUtils; /** diff --git a/marytts-runtime/src/main/java/marytts/modules/synthesis/HMMSynthesizer.java b/marytts-runtime/src/main/java/marytts/modules/synthesis/HMMSynthesizer.java index f5eb21ac8c..149cfb02b3 100644 --- a/marytts-runtime/src/main/java/marytts/modules/synthesis/HMMSynthesizer.java +++ b/marytts-runtime/src/main/java/marytts/modules/synthesis/HMMSynthesizer.java @@ -86,7 +86,7 @@ import marytts.modules.TargetFeatureLister; import marytts.modules.synthesis.Voice.Gender; import marytts.server.MaryProperties; -import marytts.unitselection.select.Target; +import marytts.features.Target; import marytts.util.MaryUtils; import marytts.util.dom.MaryDomUtils; From cbfa832a56d2d020d9fd37b460047729816ab6a8 Mon Sep 17 00:00:00 2001 From: Tristan Hamilton Date: Thu, 5 Feb 2015 14:34:49 +0100 Subject: [PATCH 09/31] getPhoneSymbol(element) method from UnitSelector copied to MaryDomUtils --- .../java/marytts/util/dom/MaryDomUtils.java | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/marytts-runtime/src/main/java/marytts/util/dom/MaryDomUtils.java b/marytts-runtime/src/main/java/marytts/util/dom/MaryDomUtils.java index f081694f7a..1870a745a3 100644 --- a/marytts-runtime/src/main/java/marytts/util/dom/MaryDomUtils.java +++ b/marytts-runtime/src/main/java/marytts/util/dom/MaryDomUtils.java @@ -196,5 +196,25 @@ public static boolean isSchemaValid(Document doc) throws MaryConfigurationExcept } return true; } + + /** + * @author hamilton + * + * method taken from UnitSelector class, used by several modules in runtime + * @param segmentOrBoundary + * @return + */ + public static String getPhoneSymbol(Element segmentOrBoundary) { + String phone; + if (segmentOrBoundary.getTagName().equals(MaryXML.PHONE)) { + phone = segmentOrBoundary.getAttribute("p"); + } else { + assert segmentOrBoundary.getTagName().equals(MaryXML.BOUNDARY) : "Expected boundary element, but got " + + segmentOrBoundary.getTagName(); + // TODO: how can we know the silence symbol here? + phone = "_"; + } + return phone; + } } From 5297f5babac4e01bdd0402fad3c391ad01bda4fd Mon Sep 17 00:00:00 2001 From: Tristan Hamilton Date: Thu, 5 Feb 2015 14:36:03 +0100 Subject: [PATCH 10/31] UnitSelector.getPhoneSymbol() replaced by MaryDomUtils.getPhoneSymbol() --- .../src/main/java/marytts/modules/AcousticModeller.java | 3 +-- .../src/main/java/marytts/modules/CARTDurationModeller.java | 2 +- .../java/marytts/modules/HalfPhoneTargetFeatureLister.java | 3 +-- .../src/main/java/marytts/modules/SoPDurationModeller.java | 3 +-- .../src/main/java/marytts/modules/TargetFeatureLister.java | 3 +-- .../src/main/java/marytts/modules/acoustic/Model.java | 4 ++-- 6 files changed, 7 insertions(+), 11 deletions(-) diff --git a/marytts-runtime/src/main/java/marytts/modules/AcousticModeller.java b/marytts-runtime/src/main/java/marytts/modules/AcousticModeller.java index 0851a40078..1093dd3f64 100644 --- a/marytts-runtime/src/main/java/marytts/modules/AcousticModeller.java +++ b/marytts-runtime/src/main/java/marytts/modules/AcousticModeller.java @@ -38,7 +38,6 @@ import marytts.modules.phonemiser.Allophone; import marytts.modules.phonemiser.AllophoneSet; import marytts.modules.synthesis.Voice; -import marytts.unitselection.select.UnitSelector; import marytts.util.MaryRuntimeUtils; import marytts.util.MaryUtils; import marytts.util.dom.MaryDomUtils; @@ -347,7 +346,7 @@ private Map> parseDocument(Document doc) throws SynthesisE segments.add(segment); // get "p" attribute... - String phone = UnitSelector.getPhoneSymbol(segment); + String phone = MaryDomUtils.getPhoneSymbol(segment); if (phone.length() == 0) { throw new SynthesisException("No phone found for segment " + segment); } diff --git a/marytts-runtime/src/main/java/marytts/modules/CARTDurationModeller.java b/marytts-runtime/src/main/java/marytts/modules/CARTDurationModeller.java index 9e9b2abff9..a3bf825444 100644 --- a/marytts-runtime/src/main/java/marytts/modules/CARTDurationModeller.java +++ b/marytts-runtime/src/main/java/marytts/modules/CARTDurationModeller.java @@ -180,7 +180,7 @@ public MaryData process(MaryData d) throws Exception { Element segmentOrBoundary; Element previous = null; while ((segmentOrBoundary = (Element) tw.nextNode()) != null) { - String phone = UnitSelector.getPhoneSymbol(segmentOrBoundary); + String phone = MaryDomUtils.getPhoneSymbol(segmentOrBoundary); Target t = new Target(phone, segmentOrBoundary); t.setFeatureVector(currentFeatureComputer.computeFeatureVector(t)); float durInSeconds; diff --git a/marytts-runtime/src/main/java/marytts/modules/HalfPhoneTargetFeatureLister.java b/marytts-runtime/src/main/java/marytts/modules/HalfPhoneTargetFeatureLister.java index 515dcefff5..ee81fad91f 100644 --- a/marytts-runtime/src/main/java/marytts/modules/HalfPhoneTargetFeatureLister.java +++ b/marytts-runtime/src/main/java/marytts/modules/HalfPhoneTargetFeatureLister.java @@ -24,7 +24,6 @@ import marytts.datatypes.MaryDataType; import marytts.datatypes.MaryXML; -import marytts.unitselection.select.UnitSelector; import marytts.features.Target; import marytts.features.HalfPhoneTarget; import marytts.util.dom.MaryDomUtils; @@ -70,7 +69,7 @@ public static List createTargetsWithPauses(List segmentsAndBoun segmentsAndBoundaries.add(finalPause); } for (Element sOrB : segmentsAndBoundaries) { - String phone = UnitSelector.getPhoneSymbol(sOrB); + String phone = MaryDomUtils.getPhoneSymbol(sOrB); targets.add(new HalfPhoneTarget(phone + "_L", sOrB, true)); targets.add(new HalfPhoneTarget(phone + "_R", sOrB, false)); } diff --git a/marytts-runtime/src/main/java/marytts/modules/SoPDurationModeller.java b/marytts-runtime/src/main/java/marytts/modules/SoPDurationModeller.java index 39123582a1..08aee21726 100644 --- a/marytts-runtime/src/main/java/marytts/modules/SoPDurationModeller.java +++ b/marytts-runtime/src/main/java/marytts/modules/SoPDurationModeller.java @@ -36,7 +36,6 @@ import marytts.modules.phonemiser.AllophoneSet; import marytts.modules.synthesis.Voice; import marytts.server.MaryProperties; -import marytts.unitselection.select.UnitSelector; import marytts.features.Target; import marytts.util.MaryRuntimeUtils; import marytts.util.MaryUtils; @@ -190,7 +189,7 @@ public MaryData process(MaryData d) throws Exception { Element segmentOrBoundary; Element previous = null; while ((segmentOrBoundary = (Element) tw.nextNode()) != null) { - String phone = UnitSelector.getPhoneSymbol(segmentOrBoundary); + String phone = MaryDomUtils.getPhoneSymbol(segmentOrBoundary); Target t = new Target(phone, segmentOrBoundary); t.setFeatureVector(currentFeatureComputer.computeFeatureVector(t)); diff --git a/marytts-runtime/src/main/java/marytts/modules/TargetFeatureLister.java b/marytts-runtime/src/main/java/marytts/modules/TargetFeatureLister.java index 5ba8b46ce5..282e4ad7a6 100644 --- a/marytts-runtime/src/main/java/marytts/modules/TargetFeatureLister.java +++ b/marytts-runtime/src/main/java/marytts/modules/TargetFeatureLister.java @@ -30,7 +30,6 @@ import marytts.features.FeatureVector; import marytts.features.TargetFeatureComputer; import marytts.modules.synthesis.Voice; -import marytts.unitselection.select.UnitSelector; import marytts.features.Target; import marytts.util.dom.MaryDomUtils; @@ -157,7 +156,7 @@ public static List createTargetsWithPauses(List segmentsAndBoun segmentsAndBoundaries.add(finalPause); } for (Element sOrB : segmentsAndBoundaries) { - String phone = UnitSelector.getPhoneSymbol(sOrB); + String phone = MaryDomUtils.getPhoneSymbol(sOrB); Target t = (Target) sOrB.getUserData("target"); if (t == null) { t = new Target(phone, sOrB); diff --git a/marytts-runtime/src/main/java/marytts/modules/acoustic/Model.java b/marytts-runtime/src/main/java/marytts/modules/acoustic/Model.java index c972490306..f3062dd1b5 100644 --- a/marytts-runtime/src/main/java/marytts/modules/acoustic/Model.java +++ b/marytts-runtime/src/main/java/marytts/modules/acoustic/Model.java @@ -31,8 +31,8 @@ import marytts.features.FeatureRegistry; import marytts.features.FeatureVector; import marytts.features.TargetFeatureComputer; -import marytts.unitselection.select.UnitSelector; import marytts.features.Target; +import marytts.util.dom.MaryDomUtils; import org.w3c.dom.Element; import org.w3c.dom.Node; @@ -250,7 +250,7 @@ protected List getTargets(List elements) { List targets = new ArrayList(elements.size()); for (Element element : elements) { assert element.getTagName() == MaryXML.PHONE; - String phone = UnitSelector.getPhoneSymbol(element); + String phone = MaryDomUtils.getPhoneSymbol(element); Target target = new Target(phone, element); targets.add(target); // compute FeatureVectors for Targets: From 6e401531b05ddbd21a90cb4f0f5ca1180ab2c3f4 Mon Sep 17 00:00:00 2001 From: Tristan Hamilton Date: Thu, 5 Feb 2015 14:36:36 +0100 Subject: [PATCH 11/31] FeatureFileReader references updated --- .../src/main/java/marytts/modules/synthesis/Voice.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/marytts-runtime/src/main/java/marytts/modules/synthesis/Voice.java b/marytts-runtime/src/main/java/marytts/modules/synthesis/Voice.java index 41154a68d9..d1f95ec4cb 100644 --- a/marytts-runtime/src/main/java/marytts/modules/synthesis/Voice.java +++ b/marytts-runtime/src/main/java/marytts/modules/synthesis/Voice.java @@ -56,6 +56,7 @@ import marytts.exceptions.SynthesisException; import marytts.features.FeatureProcessorManager; import marytts.features.FeatureRegistry; +import marytts.features.FeatureFileReader; import marytts.htsengine.HMMVoice; import marytts.modules.MaryModule; import marytts.modules.ModuleRegistry; @@ -69,7 +70,6 @@ import marytts.modules.phonemiser.AllophoneSet; import marytts.server.MaryProperties; import marytts.unitselection.UnitSelectionVoice; -import marytts.unitselection.data.FeatureFileReader; import marytts.unitselection.interpolation.InterpolatingSynthesizer; import marytts.unitselection.interpolation.InterpolatingVoice; import marytts.util.MaryRuntimeUtils; From 9369030081b009e2021581b246a463d7a5ec885d Mon Sep 17 00:00:00 2001 From: Tristan Hamilton Date: Thu, 5 Feb 2015 15:12:22 +0100 Subject: [PATCH 12/31] remove vocalization packages from marytts-builder --- .../vocalizations/HNMFeatureFileWriter.java | 232 --------- .../MLSAFeatureFileComputer.java | 397 -------------- .../vocalizations/MLSAFeatureFileWriter.java | 257 ---------- .../SnackF0ContourExtractor.java | 239 --------- .../VocalizationF0PolyFeatureFileWriter.java | 484 ------------------ .../VocalizationF0PolynomialInspector.java | 449 ---------------- .../VocalizationFeatureFileWriter.java | 278 ---------- .../VocalizationIntonationWriter.java | 435 ---------------- .../VocalizationPitchmarker.java | 160 ------ .../VocalizationTimelineMaker.java | 233 --------- .../VocalizationUnitfileWriter.java | 319 ------------ .../VocalisationLabelInspector.java | 252 --------- .../VocalizationAnnotationReader.java | 128 ----- 13 files changed, 3863 deletions(-) delete mode 100644 marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/HNMFeatureFileWriter.java delete mode 100644 marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/MLSAFeatureFileComputer.java delete mode 100644 marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/MLSAFeatureFileWriter.java delete mode 100644 marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/SnackF0ContourExtractor.java delete mode 100644 marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/VocalizationF0PolyFeatureFileWriter.java delete mode 100644 marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/VocalizationF0PolynomialInspector.java delete mode 100644 marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/VocalizationFeatureFileWriter.java delete mode 100644 marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/VocalizationIntonationWriter.java delete mode 100644 marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/VocalizationPitchmarker.java delete mode 100644 marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/VocalizationTimelineMaker.java delete mode 100644 marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/VocalizationUnitfileWriter.java delete mode 100644 marytts-builder/src/main/java/marytts/vocalizations/VocalisationLabelInspector.java delete mode 100644 marytts-builder/src/main/java/marytts/vocalizations/VocalizationAnnotationReader.java diff --git a/marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/HNMFeatureFileWriter.java b/marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/HNMFeatureFileWriter.java deleted file mode 100644 index 86a50ec887..0000000000 --- a/marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/HNMFeatureFileWriter.java +++ /dev/null @@ -1,232 +0,0 @@ -/** - * Copyright 2010 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.tools.voiceimport.vocalizations; - -import java.io.DataOutput; -import java.io.DataOutputStream; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.util.SortedMap; -import java.util.TreeMap; - -import javax.sound.sampled.AudioInputStream; -import javax.sound.sampled.AudioSystem; -import javax.sound.sampled.UnsupportedAudioFileException; - -import marytts.exceptions.MaryConfigurationException; -import marytts.signalproc.analysis.F0TrackerAutocorrelationHeuristic; -import marytts.signalproc.analysis.PitchFileHeader; -import marytts.signalproc.analysis.PitchReaderWriter; -import marytts.signalproc.analysis.RegularizedCepstrumEstimator; -import marytts.signalproc.sinusoidal.hntm.analysis.HntmAnalyzer; -import marytts.signalproc.sinusoidal.hntm.analysis.HntmAnalyzerParams; -import marytts.signalproc.sinusoidal.hntm.analysis.HntmSpeechSignal; -import marytts.signalproc.sinusoidal.hntm.synthesis.HntmSynthesizerParams; -import marytts.tools.voiceimport.DatabaseLayout; -import marytts.tools.voiceimport.VoiceImportComponent; -import marytts.util.data.MaryHeader; -import marytts.util.data.audio.AudioDoubleDataSource; -import marytts.util.io.BasenameList; -import marytts.util.math.MathUtils; -import marytts.vocalizations.HNMFeatureFileReader; -import marytts.vocalizations.VocalizationUnitFileReader; - -/** - * A component to extract and write HNM features of all vocalizations into a single file - * - * @author sathish - * - */ -public class HNMFeatureFileWriter extends VoiceImportComponent { - - private String waveExt = ".wav"; - private String ptcExt = ".ptc"; - private String hnmAnalysisFileExt = ".ana"; - private int progress = 0; - protected DatabaseLayout db = null; - protected BasenameList bnlVocalizations; - - private final String WAVEDIR = getName() + ".vocalizationWaveDir"; - private final String HNMANADIR = getName() + ".hnmAnalysisDir"; - private final String OUTHNMFILE = getName() + ".hnmAnalysisTimelineFile"; - private final String UNITFILE = getName() + ".unitFile"; - - private HntmSynthesizerParams synthesisParamsBeforeNoiseAnalysis; - private HntmAnalyzerParams analysisParams; - private PitchFileHeader f0Params; - - @Override - public SortedMap getDefaultProps(DatabaseLayout db) { - this.db = db; - if (props == null) { - props = new TreeMap(); - props.put(UNITFILE, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "files" + File.separator - + "vocalization_units" + db.getProp(db.MARYEXT)); - props.put(HNMANADIR, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "hna" + File.separator); - props.put(WAVEDIR, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "wav"); - props.put(OUTHNMFILE, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "files" + File.separator - + "vocalization_hnm_analysis" + db.getProp(db.MARYEXT)); - } - return props; - } - - @Override - protected void setupHelp() { - if (props2Help == null) { - props2Help = new TreeMap(); - props2Help.put(UNITFILE, "unit file representing all vocalizations"); - props2Help.put(HNMANADIR, "HNM features directory"); - props2Help.put(WAVEDIR, "vocalization wave files directory"); - props2Help.put(OUTHNMFILE, "a single file to write all HNM features"); - } - } - - @Override - public boolean compute() throws UnsupportedAudioFileException, IOException, MaryConfigurationException { - - VocalizationUnitFileReader listenerUnits = new VocalizationUnitFileReader(getProp(UNITFILE)); - - // sanity checker - if (listenerUnits.getNumberOfUnits() != bnlVocalizations.getLength()) { - throw new MaryConfigurationException( - "The number of vocalizations given in basename list and number of units in unit file should be matched"); - } - - F0TrackerAutocorrelationHeuristic pitchDetector = new F0TrackerAutocorrelationHeuristic(f0Params); - HntmAnalyzer ha = new HntmAnalyzer(); - DataOutputStream outStream = new DataOutputStream(new FileOutputStream(new File(getProp(OUTHNMFILE)))); - writeHeaderTo(outStream); - outStream.writeInt(listenerUnits.getNumberOfUnits()); - - for (int i = 0; i < bnlVocalizations.getLength(); i++) { - progress = i * 100 / bnlVocalizations.getLength(); - String wavFile = getProp(WAVEDIR) + File.separator + bnlVocalizations.getName(i) + waveExt; - String pitchFile = getProp(HNMANADIR) + File.separator + bnlVocalizations.getName(i) + ptcExt; - String analysisResultsFile = getProp(HNMANADIR) + File.separator + bnlVocalizations.getName(i) + hnmAnalysisFileExt; - PitchReaderWriter f0 = pitchDetector.pitchAnalyzeWavFile(wavFile, pitchFile); - AudioInputStream inputAudio = AudioSystem.getAudioInputStream(new File(wavFile)); - int samplingRate = (int) inputAudio.getFormat().getSampleRate(); - AudioDoubleDataSource signal = new AudioDoubleDataSource(inputAudio); - double[] x = signal.getAllData(); - x = MathUtils.multiply(x, 32768.0); - HntmSpeechSignal hnmSignal = ha.analyze(x, samplingRate, f0, null, analysisParams, - synthesisParamsBeforeNoiseAnalysis, analysisResultsFile); - hnmSignal.write(outStream); - } - outStream.close(); - - HNMFeatureFileReader tester = new HNMFeatureFileReader(getProp(OUTHNMFILE)); - int unitsOnDisk = tester.getNumberOfUnits(); - if (unitsOnDisk == listenerUnits.getNumberOfUnits()) { - System.out.println("Can read right number of units"); - return true; - } else { - System.out.println("Read wrong number of units: " + unitsOnDisk); - return false; - } - } - - /** - * Initialize this component - */ - @Override - protected void initialiseComp() throws Exception { - - createDirectoryifNotExists(getProp(HNMANADIR)); - - try { - String basenameFile = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "basenames.lst"; - if ((new File(basenameFile)).exists()) { - System.out.println("Loading basenames of vocalizations from '" + basenameFile + "' list..."); - bnlVocalizations = new BasenameList(basenameFile); - System.out.println("Found " + bnlVocalizations.getLength() + " vocalizations in basename list"); - } else { - String vocalWavDir = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "wav"; - System.out.println("Loading basenames of vocalizations from '" + vocalWavDir + "' directory..."); - bnlVocalizations = new BasenameList(vocalWavDir, ".wav"); - System.out.println("Found " + bnlVocalizations.getLength() + " vocalizations in " + vocalWavDir + " directory"); - } - } catch (IOException e) { - throw new MaryConfigurationException("Problem with basename list " + e); - } - f0Params = new PitchFileHeader(); - analysisParams = new HntmAnalyzerParams(); - - analysisParams.harmonicModel = HntmAnalyzerParams.HARMONICS_PLUS_NOISE; - analysisParams.noiseModel = HntmAnalyzerParams.WAVEFORM; - analysisParams.useHarmonicAmplitudesDirectly = true; - analysisParams.harmonicSynthesisMethodBeforeNoiseAnalysis = HntmSynthesizerParams.LINEAR_PHASE_INTERPOLATION; - analysisParams.regularizedCepstrumWarpingMethod = RegularizedCepstrumEstimator.REGULARIZED_CEPSTRUM_WITH_POST_MEL_WARPING; - - synthesisParamsBeforeNoiseAnalysis = new HntmSynthesizerParams(); - synthesisParamsBeforeNoiseAnalysis.harmonicPartSynthesisMethod = HntmSynthesizerParams.LINEAR_PHASE_INTERPOLATION; - } - - /** - * Create new directory if the directory doesn't exist - * - * @param dirName - * @throws Exception - */ - private void createDirectoryifNotExists(String dirName) throws Exception { - if (!(new File(dirName)).exists()) { - System.out.println(dirName + " directory does not exist; "); - if (!(new File(dirName)).mkdirs()) { - throw new Exception("Could not create directory " + dirName); - } - System.out.println("Created successfully.\n"); - } - } - - /** - * Write the header of this feature file to the given DataOutput - * - * @param out - * @throws IOException - */ - protected void writeHeaderTo(DataOutput out) throws IOException { - new MaryHeader(MaryHeader.LISTENERFEATS).writeTo(out); - } - - /** - * Return this voice import component name - */ - @Override - public String getName() { - return "HNMFeatureFileWriter"; - } - - /** - * Return the progress of this component - */ - @Override - public int getProgress() { - return this.progress; - } - - /** - * @param args - */ - public static void main(String[] args) { - // TODO Auto-generated method stub - - } -} diff --git a/marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/MLSAFeatureFileComputer.java b/marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/MLSAFeatureFileComputer.java deleted file mode 100644 index bb11c4658c..0000000000 --- a/marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/MLSAFeatureFileComputer.java +++ /dev/null @@ -1,397 +0,0 @@ -/** - * Copyright 2010 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.tools.voiceimport.vocalizations; - -import java.io.File; -import java.io.IOException; -import java.util.SortedMap; -import java.util.TreeMap; - -import marytts.exceptions.ExecutionException; -import marytts.exceptions.MaryConfigurationException; -import marytts.tools.voiceimport.DatabaseLayout; -import marytts.tools.voiceimport.VoiceImportComponent; -import marytts.util.io.BasenameList; -import marytts.util.io.FileUtils; -import marytts.util.io.General; - -/** - * MLSA feature files extractor for vocalizations. It extracts mgc features, strengths and logf0 features - * - * @author sathish - * - */ -public class MLSAFeatureFileComputer extends VoiceImportComponent { - - private String waveExt = ".wav"; - private String rawExt = ".raw"; - private String lf0Ext = ".lf0"; - private String strExt = ".str"; - private String mgcExt = ".mgc"; - private int progress = -1; - - // Default LF0 parameters - private int SAMPFREQ = 16000; - private int FRAMESHIFT = 80; - private int LOWERF0_VALUE = 100; - private int UPPERF0_VALUE = 500; - // Default MGC parameters - private int FRAMELEN = 400; - private int FFTLEN = 512; - private float FREQWARP = 0.42f; - private int MGCORDER = 24; // MGCORDER is actually 24 plus 1 include MGC[0] - // Default STR parameters - private int STRORDER = 5; - - protected DatabaseLayout db = null; - protected BasenameList bnlVocalizations; - - private final String WAVEDIR = getName() + ".vocalizationWaveDir"; - private final String MLSADIR = getName() + ".vocalizationMLSAFilesDir"; - private final String RAWDIR = getName() + ".rawFilesDir"; - private final String SCRIPTSDIR = getName() + ".scriptsDir"; - private final String TCLCOMMAND = getName() + ".tclsh-commandlinePath"; - private final String SOXCOMMAND = getName() + ".sox-commandlinePath"; - private final String SPTKPATH = getName() + ".SPTK-Path"; - private final String LOWERLF0 = getName() + ".LowerF0Value"; - private final String UPPERLF0 = getName() + ".UpperF0Value"; - - @Override - public SortedMap getDefaultProps(DatabaseLayout db) { - this.db = db; - if (props == null) { - props = new TreeMap(); - props.put(WAVEDIR, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "wav"); - props.put(SCRIPTSDIR, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "scripts"); - String mlsaDir = db.getProp(db.VOCALIZATIONSDIR) + "mlsa"; - props.put(MLSADIR, mlsaDir); - props.put(RAWDIR, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "raw"); - props.put(TCLCOMMAND, "/usr/bin/tclsh"); - props.put(SOXCOMMAND, "/usr/bin/sox"); - props.put(SPTKPATH, "/usr/bin/"); - props.put(LOWERLF0, LOWERF0_VALUE + ""); - props.put(UPPERLF0, UPPERF0_VALUE + ""); - } - return props; - } - - @Override - protected void setupHelp() { - if (props2Help == null) { - props2Help = new TreeMap(); - props2Help.put(WAVEDIR, "directory that contains vocalization wave files "); - props2Help.put(SCRIPTSDIR, "directory that contains external scripts used to compute MGC, LF0 and MGC features"); - props2Help.put(MLSADIR, "mlsa features directory"); - props2Help.put(RAWDIR, "raw files directory"); - props2Help.put(TCLCOMMAND, "tcl executable command"); - props2Help.put(SOXCOMMAND, "sox executable command"); - props2Help.put(SPTKPATH, "Path that contains SPTK executables"); - props2Help.put(LOWERLF0, "lowest pitch value specification"); - props2Help.put(UPPERLF0, "highest pitch value specification"); - } - - } - - /** - * compute logf0, mgc, strength features - */ - @Override - public boolean compute() throws Exception { - - copyFilesandScripts(); - convertWAVE2RAW(); - computeLF0Features(); - computeMGCFeatures(); - computeSTRFeatures(); - - return true; - } - - /** - * Initialize this component - * - * @throws MaryConfigurationException - * if there is problem with basename list - */ - @Override - protected void initialiseComp() throws Exception { - - createDirectoryifNotExists(getProp(MLSADIR)); - createDirectoryifNotExists(getProp(RAWDIR)); - - try { - String basenameFile = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "basenames.lst"; - if ((new File(basenameFile)).exists()) { - System.out.println("Loading basenames of vocalizations from '" + basenameFile + "' list..."); - bnlVocalizations = new BasenameList(basenameFile); - System.out.println("Found " + bnlVocalizations.getLength() + " vocalizations in basename list"); - } else { - String vocalWavDir = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "wav"; - System.out.println("Loading basenames of vocalizations from '" + vocalWavDir + "' directory..."); - bnlVocalizations = new BasenameList(vocalWavDir, ".wav"); - System.out.println("Found " + bnlVocalizations.getLength() + " vocalizations in " + vocalWavDir + " directory"); - } - } catch (IOException e) { - throw new MaryConfigurationException("Problem with basename list " + e); - } - } - - /** - * copy required scripts and files for third-party software execution - * - * @throws Exception - * if it can't copy files from source location - */ - private void copyFilesandScripts() throws Exception { - - String sep = System.getProperty("file.separator"); - - createDirectoryifNotExists(getProp(SCRIPTSDIR)); - createDirectoryifNotExists(db.getProp(db.ROOTDIR) + sep + "filters"); - - String logF0ShellScript = getProp(this.SCRIPTSDIR) + sep + "get_f0.sh"; - String logF0TCLScript = getProp(this.SCRIPTSDIR) + sep + "get_f0.tcl"; - String mgcShellScript = getProp(this.SCRIPTSDIR) + sep + "get_mgc.sh"; - String strShellScript = getProp(this.SCRIPTSDIR) + sep + "get_str.sh"; - String strTCLScript = getProp(this.SCRIPTSDIR) + sep + "get_str.tcl"; - String filterFile = db.getProp(db.ROOTDIR) + sep + "filters" + sep + "mix_excitation_filters.txt"; - String sourceDir = db.getProp(db.MARYBASE) + sep + "lib" + sep + "external" + sep + "vocalizations"; - - if (!FileUtils.exists(logF0ShellScript)) { - String sourceScript = sourceDir + sep + "scripts" + sep + "get_f0.sh"; - FileUtils.copy(sourceScript, logF0ShellScript); - (new File(logF0ShellScript)).setExecutable(true); - } - - if (!FileUtils.exists(logF0TCLScript)) { - String sourceScript = sourceDir + sep + "scripts" + sep + "get_f0.tcl"; - FileUtils.copy(sourceScript, logF0TCLScript); - (new File(logF0TCLScript)).setExecutable(true); - } - - if (!FileUtils.exists(strShellScript)) { - String sourceScript = sourceDir + sep + "scripts" + sep + "get_str.sh"; - FileUtils.copy(sourceScript, strShellScript); - (new File(strShellScript)).setExecutable(true); - } - - if (!FileUtils.exists(strTCLScript)) { - String sourceScript = sourceDir + sep + "scripts" + sep + "get_str.tcl"; - FileUtils.copy(sourceScript, strTCLScript); - (new File(strTCLScript)).setExecutable(true); - } - - if (!FileUtils.exists(mgcShellScript)) { - String sourceScript = sourceDir + sep + "scripts" + sep + "get_mgc.sh"; - FileUtils.copy(sourceScript, mgcShellScript); - (new File(mgcShellScript)).setExecutable(true); - } - - if (!FileUtils.exists(filterFile)) { - String sourceScript = sourceDir + sep + "filters" + sep + "mix_excitation_filters.txt"; - FileUtils.copy(sourceScript, filterFile); - } - } - - /** - * Convert all WAV files to RAW files to support SPTK - * - * @throws IOException - * if can't run command - * @throws ExecutionException - * if commandline throws an error - */ - private void convertWAVE2RAW() throws Exception { - - for (int i = 0; i < bnlVocalizations.getLength(); i++) { - String waveFile = getProp(WAVEDIR) + File.separator + bnlVocalizations.getName(i) + waveExt; - String rawFile = getProp(RAWDIR) + File.separator + bnlVocalizations.getName(i) + rawExt; - String command = getProp(SOXCOMMAND) + " " + waveFile + " " + rawFile; - // System.out.println( "Command: " + command ); - - try { - General.launchProc(command, "MLSAFeatureFileComputer", bnlVocalizations.getName(i)); - } catch (Exception e) { - throw new ExecutionException("\nCommand failed : " + command + "\n" + e); - } - - File rawEmptyFile = new File(rawFile); - if (rawEmptyFile.length() <= 0) { // delete all empty files - rawEmptyFile.delete(); - } - System.out.println("Creating " + bnlVocalizations.getName(i) + " RAW file"); - - File rawNFile = new File(rawFile); - if (!rawNFile.exists()) { - throw new ExecutionException("The following command failed: \n " + command + "\n"); - } - } - } - - /** - * Compute LF0 features for all RAW files using Snack and SPTK - * - * @throws IOException - * if can't run command - * @throws ExecutionException - * if commandline throws an error - * @throws ExecutionException - * if there is no output file or output file is empty - */ - private void computeLF0Features() throws Exception { - String bcCommand = "/usr/bin/bc"; - - for (int i = 0; i < bnlVocalizations.getLength(); i++) { - String rawFile = getProp(RAWDIR) + File.separator + bnlVocalizations.getName(i) + rawExt; - String lf0File = getProp(MLSADIR) + File.separator + bnlVocalizations.getName(i) + lf0Ext; - String command = getProp(this.SCRIPTSDIR) + File.separator + "get_f0.sh " + bcCommand + " " + getProp(SPTKPATH) + " " - + getProp(TCLCOMMAND) + " " + getProp(this.SCRIPTSDIR) + " " + rawFile + " " + lf0File + " " + SAMPFREQ + " " - + FRAMESHIFT + " " + getProp(LOWERLF0) + " " + getProp(UPPERLF0); - try { - General.launchProc(command, "MLSAFeatureFileComputer", bnlVocalizations.getName(i)); - } catch (Exception e) { - throw new ExecutionException("\nCommand failed : " + command + "\n" + e); - } - File lf0EmptyFile = new File(lf0File); - if (lf0EmptyFile.length() <= 0) { // delete all empty files - lf0EmptyFile.delete(); - } - // System.out.println( "Command: " + command ); - File lf0NFile = new File(lf0File); - if (!lf0NFile.exists()) { - throw new ExecutionException("The following command failed: \n " + command + "\n"); - } - System.out.println("Computed LF0 features for " + bnlVocalizations.getName(i)); - } - } - - /** - * Compute MGC features for all RAW files using SPTK - * - * @throws IOException - * if can't run command - * @throws ExecutionException - * if commandline throws an error - * @throws ExecutionException - * if there is no output file or output file is empty - */ - private void computeMGCFeatures() throws Exception { - - for (int i = 0; i < bnlVocalizations.getLength(); i++) { - String rawFile = getProp(RAWDIR) + File.separator + bnlVocalizations.getName(i) + rawExt; - String mgcFile = getProp(MLSADIR) + File.separator + bnlVocalizations.getName(i) + mgcExt; - String command = getProp(this.SCRIPTSDIR) + File.separator + "get_mgc.sh " + getProp(SPTKPATH) + " " + FRAMELEN + " " - + FRAMESHIFT + " " + FFTLEN + " " + FREQWARP + " " + MGCORDER + " " + rawFile + " " + mgcFile; - try { - General.launchProc(command, "MLSAFeatureFileComputer", bnlVocalizations.getName(i)); - } catch (Exception e) { - throw new ExecutionException("\nCommand failed : " + command + "\n" + e); - } - File lf0EmptyFile = new File(mgcFile); - if (lf0EmptyFile.length() <= 0) { // delete all empty files - lf0EmptyFile.delete(); - } - // System.out.println( "Command: " + command ); - - File lf0NFile = new File(mgcFile); - if (!lf0NFile.exists()) { - throw new ExecutionException("The following command failed: \n " + command + "\n"); - } - System.out.println("Computed MGC features for " + bnlVocalizations.getName(i)); - } - } - - /** - * Compute STRENGTH features for all RAW files using SPTK - * - * @throws IOException - * if can't run command - * @throws ExecutionException - * if commandline throws an error - * @throws ExecutionException - * if there is no output file or output file is empty - */ - private void computeSTRFeatures() throws Exception { - String bcCommand = "/usr/bin/bc"; - - for (int i = 0; i < bnlVocalizations.getLength(); i++) { - String rawFile = getProp(RAWDIR) + File.separator + bnlVocalizations.getName(i) + rawExt; - String strFile = getProp(MLSADIR) + File.separator + bnlVocalizations.getName(i) + strExt; - String command = getProp(this.SCRIPTSDIR) + File.separator + "get_str.sh " + bcCommand + " " + getProp(SPTKPATH) - + " " + getProp(TCLCOMMAND) + " " + getProp(this.SCRIPTSDIR) + " " + SAMPFREQ + " " + FRAMESHIFT + " " - + getProp(LOWERLF0) + " " + getProp(UPPERLF0) + " " + STRORDER + " " + rawFile + " " + strFile + " "; - try { - General.launchProc(command, "MLSAFeatureFileComputer", bnlVocalizations.getName(i)); - } catch (Exception e) { - throw new ExecutionException("\nCommand failed : " + command + "\n" + e); - } - File lf0EmptyFile = new File(strFile); - if (lf0EmptyFile.length() <= 0) { // delete all empty files - lf0EmptyFile.delete(); - } - // System.out.println( "Command: " + command ); - File lf0NFile = new File(strFile); - if (!lf0NFile.exists()) { - throw new ExecutionException("The following command failed: \n " + command + "\n"); - } - System.out.println("Computed Strength features for " + bnlVocalizations.getName(i)); - } - } - - /** - * Create new directory if the directory doesn't exist - * - * @param dirName - * @throws Exception - */ - private void createDirectoryifNotExists(String dirName) throws Exception { - if (!(new File(dirName)).exists()) { - System.out.println(dirName + " directory does not exist; "); - if (!(new File(dirName)).mkdirs()) { - throw new Exception("Could not create directory " + dirName); - } - System.out.println("Created successfully.\n"); - } - } - - /** - * Return this voice import component name - */ - @Override - public String getName() { - return "MLSAFeatureFileComputer"; - } - - /** - * Return the progress of this component - */ - @Override - public int getProgress() { - return this.progress; - } - - /** - * @param args - */ - public static void main(String[] args) { - // TODO Auto-generated method stub - - } -} diff --git a/marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/MLSAFeatureFileWriter.java b/marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/MLSAFeatureFileWriter.java deleted file mode 100644 index 7c16e1ec77..0000000000 --- a/marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/MLSAFeatureFileWriter.java +++ /dev/null @@ -1,257 +0,0 @@ -/** - * Copyright 2010 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.tools.voiceimport.vocalizations; - -import java.io.BufferedInputStream; -import java.io.BufferedOutputStream; -import java.io.DataOutput; -import java.io.DataOutputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.util.SortedMap; -import java.util.TreeMap; - -import marytts.exceptions.MaryConfigurationException; -import marytts.tools.voiceimport.DatabaseLayout; -import marytts.tools.voiceimport.VoiceImportComponent; -import marytts.util.data.MaryHeader; -import marytts.util.io.BasenameList; -import marytts.util.io.LEDataInputStream; -import marytts.util.math.MathUtils; -import marytts.vocalizations.MLSAFeatureFileReader; -import marytts.vocalizations.VocalizationUnitFileReader; - -/** - * A component to write all MLSA features (logf0, mgc and strengths) into a single file - * - * @author sathish - * - */ -public class MLSAFeatureFileWriter extends VoiceImportComponent { - - private String lf0Ext = ".lf0"; - private String strExt = ".str"; - private String mgcExt = ".mgc"; - private int progress = 0; - protected VocalizationUnitFileReader listenerUnits; - - private int MGCORDER = 25; // MGCORDER is actually 24 plus 1 include MGC[0] - private int STRORDER = 5; - private int LF0ORDER = 1; - - protected DatabaseLayout db = null; - protected BasenameList bnlVocalizations; - - public final String MLSADIR = getName() + ".vocalizationMLSAFilesDir"; - public final String UNITFILE = getName() + ".unitFile"; - public final String OUTMLSAFILE = getName() + ".mlsaOutputFile"; - - @Override - public SortedMap getDefaultProps(DatabaseLayout db) { - this.db = db; - if (props == null) { - props = new TreeMap(); - String mlsaDir = db.getProp(db.VOCALIZATIONSDIR) + "mlsa"; - props.put(MLSADIR, mlsaDir); - props.put(UNITFILE, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "files" + File.separator - + "vocalization_units" + db.getProp(db.MARYEXT)); - props.put(OUTMLSAFILE, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "files" + File.separator - + "vocalization_mlsa_features" + db.getProp(db.MARYEXT)); - } - return props; - } - - @Override - protected void setupHelp() { - - if (props2Help == null) { - props2Help = new TreeMap(); - props2Help.put(MLSADIR, "mlsa features directory"); - props2Help.put(UNITFILE, "unit file representing all vocalizations"); - props2Help.put(OUTMLSAFILE, "a single file to write all mlsa features"); - } - - } - - @Override - public boolean compute() throws Exception { - - listenerUnits = new VocalizationUnitFileReader(getProp(UNITFILE)); - - // write features into timeline file - DataOutputStream out = new DataOutputStream( - new BufferedOutputStream(new FileOutputStream(new File(getProp(OUTMLSAFILE))))); - writeHeaderTo(out); - writeUnitFeaturesTo(out); - out.close(); - - MLSAFeatureFileReader tester = new MLSAFeatureFileReader(getProp(OUTMLSAFILE)); - int unitsOnDisk = tester.getNumberOfUnits(); - if (unitsOnDisk == listenerUnits.getNumberOfUnits()) { - System.out.println("Can read right number of units"); - return true; - } else { - System.out.println("Read wrong number of units: " + unitsOnDisk); - return false; - } - } - - /** - * write all features into a single file - * - * @param out - * DataOutputStream - * @throws IOException - * if it can't write data into DataOutputStream - */ - private void writeUnitFeaturesTo(DataOutputStream out) throws IOException { - - int numUnits = listenerUnits.getNumberOfUnits(); - out.writeInt(numUnits); // write number of units - out.writeInt(this.LF0ORDER); // write lf0 order - out.writeInt(this.MGCORDER); // write MGC order - out.writeInt(this.STRORDER); // write STR order - - assert bnlVocalizations.getLength() == numUnits : "number of units and size of basename list should be same"; - - for (int i = 0; i < bnlVocalizations.getLength(); i++) { - - String mgcFile = getProp(MLSADIR) + File.separator + bnlVocalizations.getName(i) + mgcExt; - String lf0File = getProp(MLSADIR) + File.separator + bnlVocalizations.getName(i) + lf0Ext; - String strFile = getProp(MLSADIR) + File.separator + bnlVocalizations.getName(i) + strExt; - - LEDataInputStream lf0Data = new LEDataInputStream(new BufferedInputStream(new FileInputStream(lf0File))); - LEDataInputStream mgcData = new LEDataInputStream(new BufferedInputStream(new FileInputStream(mgcFile))); - LEDataInputStream strData = new LEDataInputStream(new BufferedInputStream(new FileInputStream(strFile))); - - int mgcSize = (int) ((new File(mgcFile)).length() / 4); // 4 bytes for float - int lf0Size = (int) ((new File(lf0File)).length() / 4); // 4 bytes for float - int strSize = (int) ((new File(strFile)).length() / 4); // 4 bytes for float - - float sizes[] = new float[3]; - int n = 0; - sizes[n++] = mgcSize / (this.MGCORDER); - sizes[n++] = lf0Size; - sizes[n++] = strSize / this.STRORDER; - int numberOfFrames = (int) MathUtils.getMin(sizes); - out.writeInt(numberOfFrames); // number of frames in this vocalization - - // first write LF0 data - int numberOfLF0Frames = numberOfFrames; - out.writeInt(numberOfLF0Frames); // number of LF0 frames - for (int j = 0; j < numberOfLF0Frames; j++) { - out.writeFloat(lf0Data.readFloat()); - } - - // second write MGC data - int numberOfMGCFrames = numberOfFrames * (this.MGCORDER); - out.writeInt(numberOfMGCFrames); // number of MGC frames - for (int j = 0; j < numberOfMGCFrames; j++) { - out.writeFloat(mgcData.readFloat()); - } - - // Third write STR data - int numberOfSTRFrames = numberOfFrames * this.STRORDER; - out.writeInt(numberOfSTRFrames); // number of MGC frames - for (int j = 0; j < numberOfSTRFrames; j++) { - out.writeFloat(strData.readFloat()); - } - } - } - - /** - * Initialize this component - * - * @throws MaryConfigurationException - * if there is problem with basename list - */ - @Override - protected void initialiseComp() throws Exception { - - String timelineDir = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "files"; - createDirectoryifNotExists(timelineDir); - - try { - String basenameFile = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "basenames.lst"; - if ((new File(basenameFile)).exists()) { - System.out.println("Loading basenames of vocalizations from '" + basenameFile + "' list..."); - bnlVocalizations = new BasenameList(basenameFile); - System.out.println("Found " + bnlVocalizations.getLength() + " vocalizations in basename list"); - } else { - String vocalWavDir = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "wav"; - System.out.println("Loading basenames of vocalizations from '" + vocalWavDir + "' directory..."); - bnlVocalizations = new BasenameList(vocalWavDir, ".wav"); - System.out.println("Found " + bnlVocalizations.getLength() + " vocalizations in " + vocalWavDir + " directory"); - } - } catch (IOException e) { - throw new MaryConfigurationException("Problem with basename list " + e); - } - } - - /** - * Write the header of this feature file to the given DataOutput - * - * @param out - * DataOutput to write header - * @throws IOException - * if it can't write data into DataOutput - */ - protected void writeHeaderTo(DataOutput out) throws IOException { - new MaryHeader(MaryHeader.LISTENERFEATS).writeTo(out); - } - - /** - * Create new directory if the directory doesn't exist - * - * @param dirName - * directory path - * @throws Exception - * if it fails - */ - private void createDirectoryifNotExists(String dirName) throws Exception { - if (!(new File(dirName)).exists()) { - System.out.println(dirName + " directory does not exist; "); - if (!(new File(dirName)).mkdirs()) { - throw new Exception("Could not create directory " + dirName); - } - System.out.println("Created successfully.\n"); - } - } - - @Override - public String getName() { - return "MLSAFeatureFileWriter"; - } - - @Override - public int getProgress() { - return this.progress; - } - - /** - * @param args - */ - public static void main(String[] args) { - // TODO Auto-generated method stub - - } -} diff --git a/marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/SnackF0ContourExtractor.java b/marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/SnackF0ContourExtractor.java deleted file mode 100644 index 41152e24be..0000000000 --- a/marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/SnackF0ContourExtractor.java +++ /dev/null @@ -1,239 +0,0 @@ -/** - * Copyright 2010 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ - -package marytts.tools.voiceimport.vocalizations; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileReader; -import java.io.FileWriter; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.PrintWriter; -import java.util.SortedMap; -import java.util.TreeMap; - -import marytts.signalproc.analysis.SPTKPitchReaderWriter; -import marytts.tools.voiceimport.DatabaseLayout; -import marytts.tools.voiceimport.VoiceImportComponent; -import marytts.util.MaryUtils; -import marytts.util.data.text.SnackTextfileDoubleDataSource; -import marytts.util.io.BasenameList; -import marytts.util.io.General; - -public class SnackF0ContourExtractor extends VoiceImportComponent { - - String vocalizationsDir; - BasenameList bnlVocalizations; - - protected DatabaseLayout db = null; - // protected String correctedPmExt = ".pm.corrected"; - protected String snackPmExt = ".snack"; - protected String lf0Ext = ".lf0"; - protected String scriptFileName; - - private int percent = 0; - - public final String MINPITCH = "SnackF0ContourExtractor.minPitch"; - public final String MAXPITCH = "SnackF0ContourExtractor.maxPitch"; - public final String COMMAND = "SnackF0ContourExtractor.command"; - public final String LF0DIR = "SnackF0ContourExtractor.pitchContoursDir"; - public final String WAVEDIR = "SnackF0ContourExtractor.vocalizationWaveDir"; - public final String SAMPLERATE = "SnackF0ContourExtractor.samplingRate"; - public final String FRAMEPERIOD = "SnackF0ContourExtractor.framePeriod"; - - public String getName() { - return "SnackF0ContourExtractor"; - } - - @Override - protected void initialiseComp() { - scriptFileName = db.getProp(db.VOCALIZATIONSDIR) + "script.snack"; - if (!(new File(getProp(LF0DIR))).exists()) { - - System.out.println("vocalizations/lf0 directory does not exist; "); - if (!(new File(getProp(LF0DIR))).mkdirs()) { - throw new Error("Could not create vocalizations/lf0"); - } - System.out.println("Created successfully.\n"); - - } - - try { - String basenameFile = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "basenames.lst"; - if ((new File(basenameFile)).exists()) { - System.out.println("Loading basenames of vocalizations from '" + basenameFile + "' list..."); - bnlVocalizations = new BasenameList(basenameFile); - System.out.println("Found " + bnlVocalizations.getLength() + " vocalizations in basename list"); - } else { - String vocalWavDir = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "wav"; - System.out.println("Loading basenames of vocalizations from '" + vocalWavDir + "' directory..."); - bnlVocalizations = new BasenameList(vocalWavDir, ".wav"); - System.out.println("Found " + bnlVocalizations.getLength() + " vocalizations in " + vocalWavDir + " directory"); - } - } catch (IOException e) { - e.printStackTrace(); - } - } - - public SortedMap getDefaultProps(DatabaseLayout db) { - this.db = db; - if (props == null) { - props = new TreeMap(); - props.put(COMMAND, "praat"); - if (db.getProp(db.GENDER).equals("female")) { - props.put(MINPITCH, "100"); - props.put(MAXPITCH, "500"); - } else { - props.put(MINPITCH, "75"); - props.put(MAXPITCH, "300"); - } - props.put(WAVEDIR, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "wav"); - props.put(LF0DIR, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "lf0"); - props.put(FRAMEPERIOD, "80"); - props.put(SAMPLERATE, "16000"); - // vocalizationsDir = db.getProp(db.ROOTDIR)+File.separator+"vocalizations"; - if (MaryUtils.isWindows()) - props.put(COMMAND, "c:/tcl/tclsh.exe"); // TODO someone with windows, please confirm or correct - else - props.put(COMMAND, "/usr/bin/tclsh"); - } - return props; - } - - /** - * The standard compute() method of the VoiceImportComponent interface. - * - * @throws InterruptedException - */ - public boolean compute() throws IOException, InterruptedException { - - String[] baseNameArray = bnlVocalizations.getListAsArray(); - System.out.println("Computing pitchmarks for " + baseNameArray.length + " utterances."); - - File script = new File(scriptFileName); - - // What is the purpose of trunk/marytts/tools/voiceimport/pm.tcl, if it's hardcoded below here? - if (script.exists()) - script.delete(); - PrintWriter toScript = new PrintWriter(new FileWriter(script)); - toScript.println("#!" + getProp(COMMAND)); - toScript.println(" "); - toScript.println("package require snack"); - toScript.println(" "); - toScript.println("snack::sound s"); - toScript.println(" "); - toScript.println("s read [lindex $argv 0]"); - toScript.println(" "); - toScript.println("set frameperiod [ lindex $argv 4 ]"); - toScript.println("set samplerate [ lindex $argv 5 ]"); - toScript.println("set framelength [expr {double($frameperiod) / $samplerate}]"); - toScript.println(" "); - toScript.println("set fd [open [lindex $argv 1] w]"); - toScript.println("puts $fd [join [s pitch -method esps -maxpitch [lindex $argv 2] -minpitch [lindex $argv 3] -framelength $framelength] \\n]"); - // toScript.println( "puts stdout [ lindex $argv 4 ]"); - // toScript.println( "puts stdout $framelength"); - toScript.println("close $fd"); - toScript.println(" "); - toScript.println("exit"); - toScript.println(" "); - toScript.close(); - - System.out.println("Computing pitchmarks for " + baseNameArray.length + " utterances."); - - /* Ensure the existence of the target pitchmark directory */ - /* - * File dir = new File(db.getProp(PMDIR)); if (!dir.exists()) { System.out.println( "Creating the directory [" + - * db.getProp(PMDIR) + "]." ); dir.mkdir(); } - */ - - /* execute snack */ - for (int i = 0; i < baseNameArray.length; i++) { - percent = 100 * i / baseNameArray.length; - String wavFile = getProp(WAVEDIR) + baseNameArray[i] + db.getProp(db.WAVEXT); - String snackFile = getProp(LF0DIR) + baseNameArray[i] + snackPmExt; - String outLF0File = getProp(LF0DIR) + baseNameArray[i] + lf0Ext; - System.out.println("Writing pm file to " + snackFile); - - boolean isWindows = true; // TODO This is WRONG, and never used. Consider removal. - String strTmp = scriptFileName + " " + wavFile + " " + snackFile + " " + getProp(MAXPITCH) + " " + getProp(MINPITCH) - + " " + getProp(FRAMEPERIOD) + " " + getProp(SAMPLERATE); - - if (MaryUtils.isWindows()) - strTmp = "cmd.exe /c " + strTmp; - else - strTmp = getProp(COMMAND) + " " + strTmp; - - General.launchProc(strTmp, "SnackF0ContourExtractor", baseNameArray[i]); - - // Now convert the snack format into EST pm format - double[] pm = new SnackTextfileDoubleDataSource(new FileReader(snackFile)).getAllData(); - SPTKPitchReaderWriter sptkPitch = new SPTKPitchReaderWriter(pm, null); - sptkPitch.writeIntoSPTKLF0File(outLF0File); - } - return true; - - } - - /** - * @param args - */ - public static void main(String[] args) { - // TODO Auto-generated method stub - - } - - public int getProgress() { - return percent; - } - - protected void setupHelp() { - if (props2Help == null) { - props2Help = new TreeMap(); - props2Help.put(COMMAND, "The command that is used to launch snack"); - props2Help.put(MINPITCH, "minimum value for the pitch (in Hz). Default: female 100, male 75"); - props2Help.put(MAXPITCH, "maximum value for the pitch (in Hz). Default: female 500, male 300"); - } - } - - private static class StreamGobbler extends Thread { - InputStream is; - String type; - - public StreamGobbler(InputStream is, String type) { - this.is = is; - this.type = type; - } - - public void run() { - try { - InputStreamReader isr = new InputStreamReader(is); - BufferedReader br = new BufferedReader(isr); - String line = null; - while ((line = br.readLine()) != null) - System.out.println(type + ">" + line); - } catch (IOException ioe) { - ioe.printStackTrace(); - } - } - } - -} diff --git a/marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/VocalizationF0PolyFeatureFileWriter.java b/marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/VocalizationF0PolyFeatureFileWriter.java deleted file mode 100644 index 33685a5492..0000000000 --- a/marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/VocalizationF0PolyFeatureFileWriter.java +++ /dev/null @@ -1,484 +0,0 @@ -/** - * Copyright 2006 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.tools.voiceimport.vocalizations; - -import java.awt.Color; -import java.io.BufferedOutputStream; -import java.io.BufferedReader; -import java.io.DataOutput; -import java.io.DataOutputStream; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.PrintWriter; -import java.io.StringReader; -import java.io.StringWriter; -import java.io.UnsupportedEncodingException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.SortedMap; -import java.util.TreeMap; - -import javax.sound.sampled.AudioFormat; -import javax.swing.JFrame; - -import marytts.exceptions.MaryConfigurationException; -import marytts.features.FeatureDefinition; -import marytts.features.FeatureVector; -import marytts.signalproc.analysis.F0TrackerAutocorrelationHeuristic; -import marytts.signalproc.analysis.PitchFileHeader; -import marytts.signalproc.display.FunctionGraph; -import marytts.tools.voiceimport.DatabaseLayout; -import marytts.tools.voiceimport.VoiceImportComponent; -import marytts.unitselection.data.FeatureFileReader; -import marytts.unitselection.data.TimelineReader; -import marytts.util.data.BufferedDoubleDataSource; -import marytts.util.data.Datagram; -import marytts.util.data.DatagramDoubleDataSource; -import marytts.util.data.MaryHeader; -import marytts.util.data.audio.AudioPlayer; -import marytts.util.data.audio.DDSAudioInputStream; -import marytts.util.math.ArrayUtils; -import marytts.util.math.Polynomial; -import marytts.util.signal.SignalProcUtils; -import marytts.vocalizations.VocalizationUnitFileReader; - -/** - * NOT COMPLETED (USEFUL FOR FUTURE) - * - * @author sathish - * - */ -public class VocalizationF0PolyFeatureFileWriter extends VoiceImportComponent { - protected File maryDir; - protected FeatureFileReader features; - protected FeatureDefinition inFeatureDefinition; - protected File outFeatureFile; - protected FeatureDefinition outFeatureDefinition; - protected VocalizationUnitFileReader listenerUnits; - protected TimelineReader audio; - protected DatabaseLayout db = null; - protected int percent = 0; - - private final String name = "F0PolynomialFeatureFileWriter"; - public final String UNITFILE = name + ".unitFile"; - public final String WAVETIMELINE = name + ".waveTimeLine"; - public final String FEATUREFILE = name + ".featureFile"; - public final String F0FEATUREFILE = name + ".f0FeatureFile"; - public final String POLYNOMORDER = name + ".polynomOrder"; - public final String SHOWGRAPH = name + ".showGraph"; - public final String INTERPOLATE = name + ".interpolate"; - public final String MINPITCH = name + ".minPitch"; - public final String MAXPITCH = name + ".maxPitch"; - - public String getName() { - return name; - } - - public SortedMap getDefaultProps(DatabaseLayout db) { - this.db = db; - if (props == null) { - props = new TreeMap(); - String fileDir = db.getProp(db.FILEDIR); - String maryExt = db.getProp(db.MARYEXT); - props.put(UNITFILE, fileDir + "halfphoneUnits" + maryExt); - props.put(WAVETIMELINE, fileDir + "timeline_waveforms" + maryExt); - props.put(FEATUREFILE, fileDir + "halfphoneFeatures" + maryExt); - props.put(F0FEATUREFILE, fileDir + "vocalizationF0Polynomials" + maryExt); - props.put(POLYNOMORDER, "3"); - props.put(SHOWGRAPH, "false"); - props.put(INTERPOLATE, "true"); - if (db.getProp(db.GENDER).equals("female")) { - props.put(MINPITCH, "100"); - props.put(MAXPITCH, "600"); - } else { - props.put(MINPITCH, "60"); - props.put(MAXPITCH, "400"); - } - } - return props; - } - - protected void setupHelp() { - if (props2Help == null) { - props2Help = new TreeMap(); - props2Help.put(UNITFILE, "file containing all halfphone units"); - props2Help.put(WAVETIMELINE, "file containing all waveforms or models that can genarate them"); - props2Help.put(FEATUREFILE, "file containing all halfphone units and their target cost features"); - props2Help.put(F0FEATUREFILE, "file containing syllable-based polynom coefficients on vowels"); - props2Help.put(POLYNOMORDER, "order of the polynoms used to approximate syllable F0 curves"); - props2Help.put(SHOWGRAPH, "whether to show a graph with f0 aproximations for each sentence"); - props2Help.put(INTERPOLATE, "whether to interpolate F0 across unvoiced regions"); - props2Help.put(MINPITCH, "minimum value for the pitch (in Hz). Default: female 100, male 75"); - props2Help.put(MAXPITCH, "maximum value for the pitch (in Hz). Default: female 500, male 300"); - } - } - - @Override - public boolean compute() throws IOException, MaryConfigurationException { - logger.info("F0 polynomial feature file writer started."); - - maryDir = new File(db.getProp(db.FILEDIR)); - if (!maryDir.exists()) { - maryDir.mkdirs(); - System.out.println("Created the output directory [" + (db.getProp(db.FILEDIR)) + "] to store the feature file."); - } - listenerUnits = new VocalizationUnitFileReader(getProp(UNITFILE)); - audio = new TimelineReader(getProp(WAVETIMELINE)); - - // features = new FeatureFileReader(getProp(FEATUREFILE)); - // inFeatureDefinition = features.getFeatureDefinition(); - StringWriter sw = new StringWriter(); - PrintWriter pw = new PrintWriter(sw); - pw.println(FeatureDefinition.BYTEFEATURES); // no byte features - pw.println(FeatureDefinition.SHORTFEATURES); // no short features - pw.println(FeatureDefinition.CONTINUOUSFEATURES); - int polynomOrder = Integer.parseInt(getProp(POLYNOMORDER)); - for (int i = polynomOrder; i >= 0; i--) { - pw.println("0 linear | f0contour_a" + i); - } - pw.close(); - String fd = sw.toString(); - logger.debug("Generated the following feature definition:"); - logger.debug(fd); - StringReader sr = new StringReader(fd); - BufferedReader br = new BufferedReader(sr); - outFeatureDefinition = new FeatureDefinition(br, true); - - outFeatureFile = new File(getProp(F0FEATUREFILE)); - DataOutputStream out = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(outFeatureFile))); - writeHeaderTo(out); - writeUnitFeaturesTo(out); - out.close(); - logger.debug("Number of processed units: " + listenerUnits.getNumberOfUnits()); - - FeatureFileReader tester = FeatureFileReader.getFeatureFileReader(getProp(F0FEATUREFILE)); - int unitsOnDisk = tester.getNumberOfUnits(); - if (unitsOnDisk == listenerUnits.getNumberOfUnits()) { - System.out.println("Can read right number of units"); - return true; - } else { - System.out.println("Read wrong number of units: " + unitsOnDisk); - return false; - } - } - - /** - * @param out - * @throws IOException - * @throws UnsupportedEncodingException - * @throws FileNotFoundException - */ - protected void writeUnitFeaturesTo(DataOutput out) throws IOException, UnsupportedEncodingException, FileNotFoundException { - int numUnits = listenerUnits.getNumberOfUnits(); - int unitSampleRate = listenerUnits.getSampleRate(); - int audioSampleRate = audio.getSampleRate(); - boolean showGraph = Boolean.parseBoolean(getProp(SHOWGRAPH)); - boolean interpolate = Boolean.parseBoolean(getProp(INTERPOLATE)); - int polynomOrder = Integer.parseInt(getProp(POLYNOMORDER)); - float[] zeros = new float[polynomOrder + 1]; - int unitIndex = 0; - - out.writeInt(numUnits); - logger.debug("Number of units : " + numUnits); - - FeatureDefinition featureDefinition = features.getFeatureDefinition(); - int fiPhoneme = featureDefinition.getFeatureIndex("phone"); - byte fvPhoneme_0 = featureDefinition.getFeatureValueAsByte(fiPhoneme, "0"); - byte fvPhoneme_Silence = featureDefinition.getFeatureValueAsByte(fiPhoneme, "_"); - int fiLR = featureDefinition.getFeatureIndex("halfphone_lr"); - byte fvLR_L = featureDefinition.getFeatureValueAsByte(fiLR, "L"); - byte fvLR_R = featureDefinition.getFeatureValueAsByte(fiLR, "R"); - int fiSylStart = featureDefinition.getFeatureIndex("segs_from_syl_start"); - int fiSylEnd = featureDefinition.getFeatureIndex("segs_from_syl_end"); - int fiSentenceStart = featureDefinition.getFeatureIndex("words_from_sentence_start"); - int fiSentenceEnd = featureDefinition.getFeatureIndex("words_from_sentence_end"); - int fiWordStart = featureDefinition.getFeatureIndex("segs_from_word_start"); - int fiWordEnd = featureDefinition.getFeatureIndex("segs_from_word_end"); - int fiVowel = featureDefinition.getFeatureIndex("ph_vc"); - byte fvVowel_Plus = featureDefinition.getFeatureValueAsByte(fiVowel, "+"); - - boolean haveUnitLogF0 = false; - int fiUnitLogF0 = -1; - int fiUnitLogF0delta = -1; - if (featureDefinition.hasFeature("unit_logf0") && featureDefinition.hasFeature("unit_logf0delta")) { - haveUnitLogF0 = true; - fiUnitLogF0 = featureDefinition.getFeatureIndex("unit_logf0"); - fiUnitLogF0delta = featureDefinition.getFeatureIndex("unit_logf0delta"); - } - - FunctionGraph f0Graph = null; - JFrame jf = null; - int iSentenceStart = -1; - int iSentenceEnd = -1; - List iSylStarts = new ArrayList(); - List iSylEnds = new ArrayList(); - List iSylVowels = new ArrayList(); - if (showGraph) { - f0Graph = new FunctionGraph(0, 1, new double[1]); - f0Graph.setYMinMax(50, 300); - f0Graph.setPrimaryDataSeriesStyle(Color.BLUE, FunctionGraph.DRAW_DOTS, FunctionGraph.DOT_FULLCIRCLE); - jf = f0Graph.showInJFrame("Sentence", false, true); - } - - for (int i = 0; i < numUnits; i++) { - percent = 100 * i / numUnits; - FeatureVector fv = features.getFeatureVector(i); - // System.out.print(featureDefinition.getFeatureValueAsString("phone", fv)); - // if (fv.getByteFeature(fiPhoneme) == fvPhoneme_0 - // || fv.getByteFeature(fiPhoneme) == fvPhoneme_Silence) continue; - if (iSentenceStart == -1 && fv.getByteFeature(fiSentenceStart) == 0 && fv.getByteFeature(fiWordStart) == 0 - && fv.getByteFeature(fiLR) == fvLR_L) { // first unit in sentence - iSentenceStart = i; - iSylStarts.clear(); - iSylEnds.clear(); - iSylVowels.clear(); - // System.out.print(", is sentence start"); - } - // Silence and edge units cannot be part of syllables, but they can - // mark start/end of sentence: - if (fv.getByteFeature(fiPhoneme) != fvPhoneme_0 && fv.getByteFeature(fiPhoneme) != fvPhoneme_Silence) { - if (fv.getByteFeature(fiSylStart) == 0 && fv.getByteFeature(fiLR) == fvLR_L) { // first segment in syllable - if (iSylStarts.size() > iSylEnds.size()) { - System.err.println("Syllable ends before other syllable starts!"); - } - iSylStarts.add(i); - // System.out.print(", is syl start"); - } - if (fv.getByteFeature(fiVowel) == fvVowel_Plus && iSylVowels.size() < iSylStarts.size()) { // first vowel unit in - // syllable - iSylVowels.add(i); - // System.out.print(", is vowel"); - } - if (fv.getByteFeature(fiSylEnd) == 0 && fv.getByteFeature(fiLR) == fvLR_R) { // last segment in syllable - iSylEnds.add(i); - // System.out.print(", is syl end"); - assert iSylStarts.size() == iSylEnds.size(); - if (iSylVowels.size() < iSylEnds.size()) { - // System.err.println("Syllable contains no vowel -- skipping"); - iSylStarts.remove(iSylStarts.size() - 1); - iSylEnds.remove(iSylEnds.size() - 1); - } - } - } - if (iSentenceStart != -1 && fv.getByteFeature(fiSentenceEnd) == 0 && fv.getByteFeature(fiWordEnd) == 0 - && fv.getByteFeature(fiLR) == fvLR_R) { // last unit in sentence - iSentenceEnd = i; - // System.out.print(", is sentence end"); - if (iSylEnds.size() < iSylStarts.size()) { - System.err.println("Last syllable in sentence is not properly closed"); - iSylEnds.add(i); - } - } - // System.out.println(); - - if (iSentenceStart >= 0 && iSentenceEnd >= iSentenceStart && iSylVowels.size() > 0) { - assert iSylStarts.size() == iSylEnds.size() : "Have " + iSylStarts.size() + " syllable starts, but " - + iSylEnds.size() + " syllable ends!"; - assert iSylStarts.size() == iSylVowels.size(); - long tsSentenceStart = listenerUnits.getUnit(iSentenceStart).startTime; - long tsSentenceEnd = listenerUnits.getUnit(iSentenceEnd).startTime + listenerUnits.getUnit(iSentenceEnd).duration; - long tsSentenceDuration = tsSentenceEnd - tsSentenceStart; - Datagram[] sentenceData = audio.getDatagrams(tsSentenceStart, tsSentenceDuration); - DatagramDoubleDataSource ddds = new DatagramDoubleDataSource(sentenceData); - double[] sentenceAudio = ddds.getAllData(); - AudioPlayer ap = null; - if (showGraph) { - ap = new AudioPlayer(new DDSAudioInputStream(new BufferedDoubleDataSource(sentenceAudio), new AudioFormat( - AudioFormat.Encoding.PCM_SIGNED, audioSampleRate, // samples per second - 16, // bits per sample - 1, // mono - 2, // nr. of bytes per frame - audioSampleRate, // nr. of frames per second - true))); // big-endian;)) - ap.start(); - } - PitchFileHeader params = new PitchFileHeader(); - params.fs = audioSampleRate; - params.minimumF0 = Double.parseDouble(getProp(MINPITCH)); - params.maximumF0 = Double.parseDouble(getProp(MAXPITCH)); - F0TrackerAutocorrelationHeuristic tracker = new F0TrackerAutocorrelationHeuristic(params); - tracker.pitchAnalyze(new BufferedDoubleDataSource(sentenceAudio)); - double frameShiftTime = tracker.getSkipSizeInSeconds(); - double[] f0Array = tracker.getF0Contour(); - if (f0Array != null) { - for (int j = 0; j < f0Array.length; j++) { - if (f0Array[j] == 0) { - f0Array[j] = Double.NaN; - } - } - if (f0Array.length >= 3) { - f0Array = SignalProcUtils.medianFilter(f0Array, 5); - } - if (showGraph) { - f0Graph.updateData(0, tsSentenceDuration / (double) audioSampleRate / f0Array.length, f0Array); - jf.repaint(); - } - - double[] f0AndInterpol; - if (interpolate) { - double[] interpol = new double[f0Array.length]; - Arrays.fill(interpol, Double.NaN); - f0AndInterpol = new double[f0Array.length]; - int iLastValid = -1; - for (int j = 0; j < f0Array.length; j++) { - if (!Double.isNaN(f0Array[j])) { // a valid value - if (iLastValid == j - 1) { - // no need to interpolate - f0AndInterpol[j] = f0Array[j]; - } else { - // need to interpolate - double prevF0; - if (iLastValid < 0) { // we don't have a previous value -- use current one - prevF0 = f0Array[j]; - } else { - prevF0 = f0Array[iLastValid]; - } - double delta = (f0Array[j] - prevF0) / (j - iLastValid); - double f0 = prevF0; - for (int k = iLastValid + 1; k < j; k++) { - f0 += delta; - interpol[k] = f0; - f0AndInterpol[k] = f0; - } - } - iLastValid = j; - } - } - if (showGraph) { - f0Graph.addDataSeries(interpol, Color.GREEN, FunctionGraph.DRAW_DOTS, FunctionGraph.DOT_EMPTYCIRCLE); - jf.repaint(); - } - } else { - f0AndInterpol = f0Array.clone(); - } - - for (int j = 0; j < f0AndInterpol.length; j++) { - if (f0AndInterpol[j] == 0) - f0AndInterpol[j] = Double.NaN; - else - f0AndInterpol[j] = Math.log(f0AndInterpol[j]); - } - double[] approx = new double[f0Array.length]; - Arrays.fill(approx, Double.NaN); - for (int s = 0; s < iSylStarts.size(); s++) { - long tsSylStart = listenerUnits.getUnit(iSylStarts.get(s)).startTime; - long tsSylEnd = listenerUnits.getUnit(iSylEnds.get(s)).startTime - + listenerUnits.getUnit(iSylEnds.get(s)).duration; - long tsSylDuration = tsSylEnd - tsSylStart; - int iSylVowel = iSylVowels.get(s); - // now map time to position in f0AndInterpol array: - int iSylStart = (int) (((double) (tsSylStart - tsSentenceStart) / tsSentenceDuration) * f0AndInterpol.length); - assert iSylStart >= 0; - int iSylEnd = iSylStart + (int) ((double) tsSylDuration / tsSentenceDuration * f0AndInterpol.length) + 1; - if (iSylEnd > f0AndInterpol.length) - iSylEnd = f0AndInterpol.length; - // System.out.println("Syl "+s+" from "+iSylStart+" to "+iSylEnd+" out of "+f0AndInterpol.length); - double[] sylF0 = new double[iSylEnd - iSylStart]; - System.arraycopy(f0AndInterpol, iSylStart, sylF0, 0, sylF0.length); - double[] coeffs = Polynomial.fitPolynomial(sylF0, polynomOrder); - if (coeffs != null) { - if (showGraph) { - double[] sylPred = Polynomial.generatePolynomialValues(coeffs, sylF0.length, 0, 1); - System.arraycopy(sylPred, 0, approx, iSylStart, sylPred.length); - } - // Write coefficients to file - while (unitIndex < iSylVowel) { - FeatureVector outFV = outFeatureDefinition.toFeatureVector(unitIndex, null, null, zeros); - outFV.writeTo(out); - unitIndex++; - } - float[] fcoeffs = ArrayUtils.copyDouble2Float(coeffs); - // System.out.print("Polynomial values (unit "+unitIndex+") "); - // for (int p=0; p. - * - */ -package marytts.tools.voiceimport.vocalizations; - -import java.awt.Color; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileWriter; -import java.io.IOException; -import java.io.PrintWriter; -import java.io.UnsupportedEncodingException; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.Set; -import java.util.SortedMap; -import java.util.TreeMap; - -import javax.sound.sampled.AudioFormat; -import javax.sound.sampled.AudioInputStream; -import javax.sound.sampled.AudioSystem; -import javax.sound.sampled.UnsupportedAudioFileException; -import javax.swing.JFrame; - -import marytts.features.FeatureDefinition; -import marytts.signalproc.analysis.F0TrackerAutocorrelationHeuristic; -import marytts.signalproc.analysis.PitchFileHeader; -import marytts.signalproc.analysis.PitchReaderWriter; -import marytts.signalproc.analysis.SPTKPitchReaderWriter; -import marytts.signalproc.analysis.distance.DistanceComputer; -import marytts.signalproc.display.FunctionGraph; -import marytts.tools.voiceimport.DatabaseLayout; -import marytts.tools.voiceimport.VoiceImportComponent; -import marytts.unitselection.data.FeatureFileReader; -import marytts.unitselection.data.TimelineReader; -import marytts.unitselection.data.UnitFileReader; -import marytts.util.data.BufferedDoubleDataSource; -import marytts.util.data.audio.AudioDoubleDataSource; -import marytts.util.data.audio.AudioPlayer; -import marytts.util.data.audio.DDSAudioInputStream; -import marytts.util.io.BasenameList; -import marytts.util.math.Polynomial; -import marytts.util.signal.SignalProcUtils; -import marytts.vocalizations.KMeansClusterer; - -public class VocalizationF0PolynomialInspector extends VoiceImportComponent { - protected FeatureFileReader features; - protected FeatureDefinition inFeatureDefinition; - protected UnitFileReader units; - protected FeatureFileReader contours; - protected TimelineReader audio; - protected DatabaseLayout db = null; - protected int percent = 0; - protected FunctionGraph f0Graph = null; - protected JFrame jf = null; - protected PrintWriter featurePW; - protected double costMeasure = 0; - private HashMap minF0Values; - private HashMap maxF0Values; - private Set characters; - - protected BasenameList bnlVocalizations; - - private final String name = "VocalizationF0PolynomialInspector"; - public final String WAVEDIR = name + ".waveDir"; - public final String F0POLYFILE = name + ".f0PolynomialFeatureFile"; - public final String F0MIN = name + ".f0Minimum"; - public final String F0MAX = name + ".f0Maximum"; - public final String PARTBASENAME = name + ".partBaseName"; - public final String ONEWORD = name + ".oneWordDescription"; - public final String KCLUSTERS = name + ".numberOfClusters"; - public final String POLYORDER = name + ".polynomialOrder"; - public final String ISEXTERNALF0 = name + ".isExternalF0Usage"; - public final String EXTERNALF0FORMAT = name + ".externalF0Format"; - public final String EXTERNALDIR = name + ".externalF0Directory"; - public final String EXTERNALEXT = name + ".externalF0Extention"; - - public String getName() { - return name; - } - - public SortedMap getDefaultProps(DatabaseLayout db) { - this.db = db; - if (props == null) { - props = new TreeMap(); - props.put(WAVEDIR, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "wav"); - props.put(F0POLYFILE, "VocalizationF0PolyFeatureFile.txt"); - props.put(PARTBASENAME, ""); - props.put(ONEWORD, ""); - props.put(F0MIN, "50"); - props.put(F0MAX, "500"); - props.put(KCLUSTERS, "15"); - props.put(POLYORDER, "3"); - props.put(ISEXTERNALF0, "true"); - props.put(EXTERNALF0FORMAT, "sptk"); - props.put(EXTERNALDIR, "lf0"); - props.put(EXTERNALEXT, ".lf0"); - } - return props; - } - - protected void setupHelp() { - if (props2Help == null) { - props2Help = new TreeMap(); - props2Help.put(WAVEDIR, "dir containing all waveforms "); - } - } - - @Override - protected void initialiseComp() { - minF0Values = new HashMap(); - maxF0Values = new HashMap(); - minF0Values.put("Spike", 50); - minF0Values.put("Poppy", 170); - minF0Values.put("Obadiah", 70); - minF0Values.put("Prudence", 130); - maxF0Values.put("Spike", 150); - maxF0Values.put("Poppy", 380); - maxF0Values.put("Obadiah", 150); - maxF0Values.put("Prudence", 280); - - characters = new HashSet(); - characters.add("Spike"); - characters.add("Poppy"); - characters.add("Obadiah"); - characters.add("Prudence"); - - try { - String basenameFile = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "basenames.lst"; - if ((new File(basenameFile)).exists()) { - System.out.println("Loading basenames of vocalizations from '" + basenameFile + "' list..."); - bnlVocalizations = new BasenameList(basenameFile); - System.out.println("Found " + bnlVocalizations.getLength() + " vocalizations in basename list"); - } else { - String vocalWavDir = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "wav"; - System.out.println("Loading basenames of vocalizations from '" + vocalWavDir + "' directory..."); - bnlVocalizations = new BasenameList(vocalWavDir, ".wav"); - System.out.println("Found " + bnlVocalizations.getLength() + " vocalizations in " + vocalWavDir + " directory"); - } - } catch (IOException e) { - e.printStackTrace(); - } - } - - public boolean compute() throws IOException, UnsupportedAudioFileException { - logger.info("F0 polynomial feature file writer started."); - f0Graph = new FunctionGraph(0, 1, new double[1]); - f0Graph.setYMinMax(50, 550); - f0Graph.setPrimaryDataSeriesStyle(Color.BLUE, FunctionGraph.DRAW_DOTS, FunctionGraph.DOT_FULLCIRCLE); - jf = f0Graph.showInJFrame("Sentence", false, true); - - String outPutFile = db.getProp(db.ROOTDIR) + File.separator + getProp(ONEWORD) + getProp(PARTBASENAME) - + getProp(F0POLYFILE); - featurePW = new PrintWriter(new FileWriter(new File(outPutFile))); - - for (int i = 0; i < bnlVocalizations.getLength(); i++) { - percent = 100 * i / bnlVocalizations.getLength(); - displaySentences(bnlVocalizations.getName(i)); - - } - featurePW.flush(); - featurePW.close(); - System.out.println("Total Cost : " + costMeasure / (double) bnlVocalizations.getLength()); - - // String fileName = "/home/sathish/phd/voices/en-GB-listener/vocal-polynomials/SpiVocalizationF0PolyFeatureFile.txt"; - int kValue = (new Integer(getProp(KCLUSTERS))).intValue(); - KMeansClusterer kmc = new KMeansClusterer(); - kmc.loadF0Polynomials(outPutFile); - kmc.trainer(kValue); - // System.exit(0); - - return true; - } - - /** - * @param out - * @throws IOException - * @throws UnsupportedAudioFileException - * @throws UnsupportedEncodingException - * @throws FileNotFoundException - */ - protected void displaySentences(String baseName) throws IOException, UnsupportedAudioFileException { - /* - * int numUnits = units.getNumberOfUnits(); int unitSampleRate = units.getSampleRate(); int audioSampleRate = - * audio.getSampleRate(); int unitIndex = 0; - * - * logger.debug("Number of units : "+numUnits); - */ - String partBaseName = getProp(PARTBASENAME).trim(); - if (!"".equals(partBaseName) && !baseName.contains(partBaseName)) { - return; - } - - /* - * String targetDescription = getProp(ONEWORD).trim(); String textFileName = - * db.getProp(db.TEXTDIR)+File.separator+baseName+db.getProp(db.TEXTEXT); String audioDescriprion = - * FileUtils.getFileAsString(new File(textFileName), "UTF-8"); - * - * if ( !"".equals(targetDescription) && !targetDescription.equals(audioDescriprion.trim())) { return; } - */ - - // if ( !baseName.equals("") && !baseName.contains(getProp(PARTBASENAME)) ){ - // return; - // } - - String waveFile = getProp(WAVEDIR) + File.separator + baseName + db.getProp(db.WAVEXT); - AudioInputStream inputAudio = AudioSystem.getAudioInputStream(new File(waveFile)); - - // Enforce PCM_SIGNED encoding - if (!inputAudio.getFormat().getEncoding().equals(AudioFormat.Encoding.PCM_SIGNED)) { - inputAudio = AudioSystem.getAudioInputStream(AudioFormat.Encoding.PCM_SIGNED, inputAudio); - } - - int audioSampleRate = (int) inputAudio.getFormat().getSampleRate(); - AudioDoubleDataSource signal = new AudioDoubleDataSource(inputAudio); - double[] sentenceAudio = signal.getAllData(); // Copies all samples in wav file into a double buffer - long tsSentenceDuration = sentenceAudio.length; - - /* - * Datagram[] sentenceData = audio.getDatagrams(tsSentenceStart, tsSentenceDuration); DatagramDoubleDataSource ddds = new - * DatagramDoubleDataSource(sentenceData); double[] sentenceAudio = ddds.getAllData(); - */ - - AudioPlayer ap = null; - ap = new AudioPlayer(new DDSAudioInputStream(new BufferedDoubleDataSource(sentenceAudio), new AudioFormat( - AudioFormat.Encoding.PCM_SIGNED, audioSampleRate, // samples per second - 16, // bits per sample - 1, // mono - 2, // nr. of bytes per frame - audioSampleRate, // nr. of frames per second - true))); // big-endian;)) - ap.start(); - - PitchFileHeader params = new PitchFileHeader(); - // params.minimumF0 = 50; - // params.maximumF0 = 250; - // params.minimumF0 = (new Double(getProp(F0MIN))).doubleValue(); - // params.maximumF0 = (new Double(getProp(F0MAX))).doubleValue(); - String character = getCharacterName(baseName); - params.minimumF0 = (minF0Values.get(character)).doubleValue(); - params.maximumF0 = (maxF0Values.get(character)).doubleValue(); - params.fs = audioSampleRate; - - double[] f0Array = null; - - if ("true".equals(getProp(ISEXTERNALF0))) { - - String externalFormat = getProp(EXTERNALF0FORMAT); - String externalDir = getProp(EXTERNALDIR); - String externalExt = getProp(EXTERNALEXT); - - if ("sptk".equals(externalFormat)) { - String fileName = db.getProp(db.VOCALIZATIONSDIR) + File.separator + externalDir + File.separator + baseName - + externalExt; - SPTKPitchReaderWriter sprw = new SPTKPitchReaderWriter(fileName); - f0Array = sprw.getF0Contour(); - } else if ("ptc".equals(externalFormat)) { - String fileName = db.getProp(db.ROOTDIR) + File.separator + externalDir + File.separator + baseName + externalExt; - PitchReaderWriter sprw = new PitchReaderWriter(fileName); - f0Array = sprw.contour; - } - } else { - F0TrackerAutocorrelationHeuristic tracker = new F0TrackerAutocorrelationHeuristic(params); - tracker.pitchAnalyze(new BufferedDoubleDataSource(sentenceAudio)); - // double frameShiftTime = tracker.getSkipSizeInSeconds(); - f0Array = tracker.getF0Contour(); - } - - // f0Array = cutStartEndUnvoicedSegments(f0Array); - - if (f0Array != null) { - for (int j = 0; j < f0Array.length; j++) { - if (f0Array[j] == 0) { - f0Array[j] = Double.NaN; - } - } - if (f0Array.length >= 3) { - f0Array = SignalProcUtils.medianFilter(f0Array, 5); - } - f0Graph.updateData(0, tsSentenceDuration / (double) audioSampleRate / f0Array.length, f0Array); - jf.repaint(); - - double[] f0AndInterpol; - double[] interpol = new double[f0Array.length]; - Arrays.fill(interpol, Double.NaN); - f0AndInterpol = new double[f0Array.length]; - int iLastValid = -1; - for (int j = 0; j < f0Array.length; j++) { - if (!Double.isNaN(f0Array[j])) { // a valid value - if (iLastValid == j - 1) { - // no need to interpolate - f0AndInterpol[j] = f0Array[j]; - } else { - // need to interpolate - double prevF0; - if (iLastValid < 0) { // we don't have a previous value -- use current one - prevF0 = f0Array[j]; - } else { - prevF0 = f0Array[iLastValid]; - } - double delta = (f0Array[j] - prevF0) / (j - iLastValid); - double f0 = prevF0; - for (int k = iLastValid + 1; k < j; k++) { - f0 += delta; - interpol[k] = f0; - f0AndInterpol[k] = f0; - } - } - iLastValid = j; - } - } - f0Graph.addDataSeries(interpol, Color.GREEN, FunctionGraph.DRAW_DOTS, FunctionGraph.DOT_EMPTYCIRCLE); - jf.repaint(); - - double[] f0AndInterpolate = combineF0andInterpolate(f0Array, interpol); - - // double[] coeffs = Polynomial.fitPolynomial(sylF0, polynomOrder); - int polynomialOrder = (new Integer(getProp(POLYORDER))).intValue(); - double[] coeffs = Polynomial.fitPolynomial(f0AndInterpolate, polynomialOrder); - - if (coeffs != null) { - double[] sylPred = Polynomial.generatePolynomialValues(coeffs, interpol.length, 0, 1); - f0Graph.addDataSeries(sylPred, Color.RED, FunctionGraph.DRAW_LINE, -1); - double eqDistance = DistanceComputer.getEuclideanDistance(sylPred, f0AndInterpolate); - System.out.println(baseName + " - EqDist: " + eqDistance / (double) sylPred.length); - costMeasure += (eqDistance / (double) sylPred.length); - featurePW.print(baseName + " "); - for (double c : coeffs) { - featurePW.print(c + " "); - } - featurePW.println(); - - } - - // double[] pred = Polynomial.generatePolynomialValues(coeffs, iUnitDurationInArray, 0, 1); - // f0Graph.addDataSeries(unitF0, Color.BLACK, FunctionGraph.DRAW_LINE, -1); - - try { - ap.join(); - Thread.sleep(10); - } catch (InterruptedException ie) { - } - // System.out.println(); - - } - - } - - private double[] cutStartEndUnvoicedSegments(double[] array) { - - if (array == null) - return null; - - int startIndex = 0; - int endIndex = array.length; - - // find start index - for (int i = 0; i < array.length; i++) { - if (array[i] != 0) { - startIndex = i; - break; - } - } - - // find end index - for (int i = (array.length - 1); i > 0; i--) { - if (array[i] != 0) { - endIndex = i; - break; - } - } - - int newArraySize = endIndex - startIndex; - - double[] newArray = new double[newArraySize]; - System.arraycopy(array, startIndex, newArray, 0, newArraySize); - - for (int i = 0; i < newArray.length; i++) { - System.out.println(newArray[i]); - } - System.out.println("Resized from " + array.length + " to " + newArraySize); - - return newArray; - } - - private String getCharacterName(String baseName) { - Iterator it = characters.iterator(); - while (it.hasNext()) { - String character = it.next().trim(); - if (baseName.startsWith(character)) { - return character; - } - } - return null; - } - - private double[] combineF0andInterpolate(double[] f0Array, double[] interpol) { - - double[] f0AndInterpolate = new double[f0Array.length]; - Arrays.fill(f0AndInterpolate, Double.NaN); - for (int i = 0; i < f0Array.length; i++) { - if (!Double.isNaN(f0Array[i])) { - f0AndInterpolate[i] = f0Array[i]; - } else if (!Double.isNaN(interpol[i])) { - f0AndInterpolate[i] = interpol[i]; - } - // System.out.println(f0Array[i]+" "+interpol[i]+" "+f0AndInterpolate[i]); - } - - return f0AndInterpolate; - } - - /** - * Provide the progress of computation, in percent, or -1 if that feature is not implemented. - * - * @return -1 if not implemented, or an integer between 0 and 100. - */ - public int getProgress() { - return percent; - } - - /** - * @param args - */ - public static void main(String[] args) throws Exception { - VocalizationF0PolynomialInspector acfeatsWriter = new VocalizationF0PolynomialInspector(); - DatabaseLayout db = new DatabaseLayout(acfeatsWriter); - acfeatsWriter.compute(); - } - -} diff --git a/marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/VocalizationFeatureFileWriter.java b/marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/VocalizationFeatureFileWriter.java deleted file mode 100644 index f1d4558671..0000000000 --- a/marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/VocalizationFeatureFileWriter.java +++ /dev/null @@ -1,278 +0,0 @@ -/** - * Copyright 2006 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.tools.voiceimport.vocalizations; - -import java.io.BufferedOutputStream; -import java.io.BufferedReader; -import java.io.DataOutput; -import java.io.DataOutputStream; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.FileReader; -import java.io.IOException; -import java.io.UnsupportedEncodingException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Map; -import java.util.SortedMap; -import java.util.TreeMap; - -import marytts.exceptions.MaryConfigurationException; -import marytts.features.FeatureDefinition; -import marytts.features.FeatureVector; -import marytts.tools.voiceimport.DatabaseLayout; -import marytts.tools.voiceimport.VoiceImportComponent; -import marytts.util.data.MaryHeader; -import marytts.util.io.BasenameList; -import marytts.vocalizations.VocalizationAnnotationReader; -import marytts.vocalizations.VocalizationFeatureFileReader; -import marytts.vocalizations.VocalizationUnitFileReader; - -/** - * vocalization feature file writer - * - * @author sathish - * - */ -public class VocalizationFeatureFileWriter extends VoiceImportComponent { - - protected File outFeatureFile; - protected FeatureDefinition featureDefinition; - protected VocalizationUnitFileReader listenerUnits; - - protected DatabaseLayout db = null; - protected int percent = 0; - private ArrayList featureCategories; // feature categories - private Map> annotationData; // basename --> (feature category, feature value) - - protected String vocalizationsDir; - protected BasenameList bnlVocalizations; - - private final String name = "VocalizationFeatureFileWriter"; - public final String UNITFILE = name + ".unitFile"; - public final String FEATUREFILE = name + ".featureFile"; - public final String MANUALFEATURES = name + ".annotationFeatureFile"; - public final String FEATDEF = name + ".featureDefinition"; - - public final String POLYNOMORDER = name + ".polynomOrder"; - public final String SHOWGRAPH = name + ".showGraph"; - public final String INTERPOLATE = name + ".interpolate"; - public final String MINPITCH = name + ".minPitch"; - public final String MAXPITCH = name + ".maxPitch"; - - // public String BASELIST = name + ".backchannelBNL"; - - public String getName() { - return name; - } - - public SortedMap getDefaultProps(DatabaseLayout db) { - this.db = db; - if (props == null) { - props = new TreeMap(); - String fileDir = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "files" + File.separator; - String maryExt = db.getProp(db.MARYEXT); - props.put(UNITFILE, fileDir + "vocalization_units" + maryExt); - props.put(FEATUREFILE, fileDir + "vocalization_features" + maryExt); - props.put(MANUALFEATURES, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "features" + File.separator - + "annotation_vocalizations_features.txt"); - props.put(FEATDEF, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "features" + File.separator - + "vocalization_feature_definition.txt"); - } - return props; - } - - protected void setupHelp() { - if (props2Help == null) { - props2Help = new TreeMap(); - props2Help.put(UNITFILE, "file containing all halfphone units"); - props2Help.put(FEATUREFILE, "file containing all halfphone units and their target cost features"); - } - } - - @Override - protected void initialiseComp() { - - featureCategories = new ArrayList(); // feature categories - annotationData = new HashMap>(); // basename --> (feature category, feature value) - - try { - String basenameFile = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "basenames.lst"; - if ((new File(basenameFile)).exists()) { - System.out.println("Loading basenames of vocalizations from '" + basenameFile + "' list..."); - bnlVocalizations = new BasenameList(basenameFile); - System.out.println("Found " + bnlVocalizations.getLength() + " vocalizations in basename list"); - } else { - String vocalWavDir = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "wav"; - System.out.println("Loading basenames of vocalizations from '" + vocalWavDir + "' directory..."); - bnlVocalizations = new BasenameList(vocalWavDir, ".wav"); - System.out.println("Found " + bnlVocalizations.getLength() + " vocalizations in " + vocalWavDir + " directory"); - } - } catch (IOException e) { - e.printStackTrace(); - } - } - - @Override - public boolean compute() throws IOException, MaryConfigurationException { - // read feature definition - BufferedReader fDBfr = new BufferedReader(new FileReader(new File(getProp(FEATDEF)))); - // featureDefinition = new FeatureDefinition(fDBfr, true); - featureDefinition = new FeatureDefinition(fDBfr, true); - listenerUnits = new VocalizationUnitFileReader(getProp(UNITFILE)); - - // load annotation - VocalizationAnnotationReader annotationReader = new VocalizationAnnotationReader(getProp(MANUALFEATURES), - bnlVocalizations); - annotationData = annotationReader.getVocalizationsAnnotation(); - featureCategories = annotationReader.getFeatureList(); - - // write features into timeline file - DataOutputStream out = new DataOutputStream( - new BufferedOutputStream(new FileOutputStream(new File(getProp(FEATUREFILE))))); - writeHeaderTo(out); - writeUnitFeaturesTo(out); - out.close(); - logger.debug("Number of processed units: " + listenerUnits.getNumberOfUnits()); - - VocalizationFeatureFileReader tester = new VocalizationFeatureFileReader(getProp(FEATUREFILE)); - int unitsOnDisk = tester.getNumberOfUnits(); - if (unitsOnDisk == listenerUnits.getNumberOfUnits()) { - System.out.println("Can read right number of units"); - return true; - } else { - System.out.println("Read wrong number of units: " + unitsOnDisk); - return false; - } - } - - protected void writeUnitFeaturesTo(DataOutput out) throws IOException, UnsupportedEncodingException, FileNotFoundException { - - int numUnits = listenerUnits.getNumberOfUnits(); - - out.writeInt(numUnits); - // logger.debug("Number of vocalization units : "+numUnits); - System.out.println("Number of vocalization units : " + numUnits); - System.out.println("Annotation number of units: " + annotationData.size()); - if (annotationData.size() != listenerUnits.getNumberOfUnits()) { - throw new RuntimeException("Number of units in vocalizations unit-file is not equal to number of basenames. "); - } - - if (featureCategories.size() != featureDefinition.getNumberOfFeatures()) { - throw new RuntimeException( - "Number of categories in feature_definition is not equal to features given in annotation file "); - } - - /** - * TODO sanity check for each basename - */ - - int noOfFeatures = featureDefinition.getNumberOfFeatures(); - String[] featureNames = new String[noOfFeatures]; - for (int i = 0; i < featureNames.length; i++) { - featureNames[i] = featureDefinition.getFeatureName(i); - } - - for (int i = 0; i < bnlVocalizations.getLength(); i++) { - - byte[] byteFeatures; - short[] shortFeatures; - float[] continiousFeatures; - - String baseName = bnlVocalizations.getName(i); - Map singleAnnotation = annotationData.get(baseName); - - int noByteFeatures = featureDefinition.getNumberOfByteFeatures(); - int noShortFeatures = featureDefinition.getNumberOfShortFeatures(); - int noContiniousFeatures = featureDefinition.getNumberOfContinuousFeatures(); - - // create features array - byteFeatures = new byte[noByteFeatures]; - shortFeatures = new short[noShortFeatures]; - continiousFeatures = new float[noContiniousFeatures]; - - int countByteFeatures = 0; - int countShortFeatures = 0; - int countFloatFeatures = 0; - - for (int j = 0; j < featureNames.length; j++) { - String fName = featureNames[j]; - if (featureDefinition.isByteFeature(j)) { // Byte feature - if (singleAnnotation.containsKey(fName)) { - byteFeatures[countByteFeatures++] = featureDefinition.getFeatureValueAsByte(j, - singleAnnotation.get(fName)); - } else { - byteFeatures[countByteFeatures++] = (byte) 0; - } - } else if (featureDefinition.isShortFeature(j)) { // Short feature - if (singleAnnotation.containsKey(fName)) { - shortFeatures[countShortFeatures++] = featureDefinition.getFeatureValueAsShort(j, - singleAnnotation.get(fName)); - } else { - shortFeatures[countShortFeatures++] = (short) 0; - } - } else if (featureDefinition.isContinuousFeature(j)) { // Continuous feature - if (!singleAnnotation.containsKey(fName)) { - continiousFeatures[countFloatFeatures++] = Float.NaN; - } else if ("NRI".equals(singleAnnotation.get(fName))) { - continiousFeatures[countFloatFeatures++] = Float.NaN; - } else { - continiousFeatures[countFloatFeatures++] = (new Float(singleAnnotation.get(fName))).floatValue(); - } - } - } - - FeatureVector outFV = featureDefinition.toFeatureVector(i, byteFeatures, shortFeatures, continiousFeatures); - outFV.writeTo(out); - } - - } - - /** - * Write the header of this feature file to the given DataOutput - * - * @param out - * @throws IOException - */ - protected void writeHeaderTo(DataOutput out) throws IOException { - new MaryHeader(MaryHeader.LISTENERFEATS).writeTo(out); - featureDefinition.writeBinaryTo(out); - } - - /** - * Provide the progress of computation, in percent, or -1 if that feature is not implemented. - * - * @return -1 if not implemented, or an integer between 0 and 100. - */ - public int getProgress() { - return percent; - } - - /** - * @param args - */ - public static void main(String[] args) throws Exception { - VocalizationFeatureFileWriter acfeatsWriter = new VocalizationFeatureFileWriter(); - DatabaseLayout db = new DatabaseLayout(acfeatsWriter); - acfeatsWriter.compute(); - } - -} diff --git a/marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/VocalizationIntonationWriter.java b/marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/VocalizationIntonationWriter.java deleted file mode 100644 index 590f56e031..0000000000 --- a/marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/VocalizationIntonationWriter.java +++ /dev/null @@ -1,435 +0,0 @@ -/** - * Copyright 2010 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ - -package marytts.tools.voiceimport.vocalizations; - -import java.io.BufferedOutputStream; -import java.io.DataOutput; -import java.io.DataOutputStream; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.util.Arrays; -import java.util.SortedMap; -import java.util.TreeMap; - -import javax.sound.sampled.AudioFormat; -import javax.sound.sampled.AudioInputStream; -import javax.sound.sampled.AudioSystem; -import javax.sound.sampled.UnsupportedAudioFileException; - -import marytts.exceptions.MaryConfigurationException; -import marytts.signalproc.analysis.F0TrackerAutocorrelationHeuristic; -import marytts.signalproc.analysis.PitchFileHeader; -import marytts.signalproc.analysis.PitchReaderWriter; -import marytts.signalproc.analysis.SPTKPitchReaderWriter; -import marytts.tools.voiceimport.DatabaseLayout; -import marytts.tools.voiceimport.VoiceImportComponent; -import marytts.util.data.BufferedDoubleDataSource; -import marytts.util.data.MaryHeader; -import marytts.util.data.audio.AudioDoubleDataSource; -import marytts.util.io.BasenameList; -import marytts.util.math.Polynomial; -import marytts.util.signal.SignalProcUtils; -import marytts.vocalizations.VocalizationIntonationReader; -import marytts.vocalizations.VocalizationUnitFileReader; - -/** - * Vocalization intonation writer into a time-line file This class can create a timeline file with intonation contours and thier - * polynomial coeffs - * - * @author sathish pammi - * - */ -public class VocalizationIntonationWriter extends VoiceImportComponent { - - protected String vocalizationsDir; - protected BasenameList bnlVocalizations; - protected VocalizationUnitFileReader listenerUnits; - - protected DatabaseLayout db = null; - protected int percent = 0; - - public final String PITCHDIR = getName() + ".pitchDir"; - public final String WAVEDIR = getName() + ".inputWaveDir"; - public final String POLYORDER = getName() + ".polynomialOrder"; - public final String ISEXTERNALF0 = getName() + ".isExternalF0Usage"; - public final String EXTERNALF0FORMAT = getName() + ".externalF0Format"; - public final String EXTERNALEXT = getName() + ".externalF0Extention"; - public final String UNITFILE = getName() + ".unitFile"; - public final String SKIPSIZE = getName() + ".skipSize"; - public final String WINDOWSIZE = getName() + ".windowSize"; - public final String F0TIMELINE = getName() + ".intonationTimeLineFile"; - public final String F0FEATDEF = getName() + ".intonationFeatureDefinition"; - - public String getName() { - return "VocalizationIntonationWriter"; - } - - @Override - protected void initialiseComp() { - - String timelineDir = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "files"; - if (!(new File(timelineDir)).exists()) { - System.out.println("vocalizations/files directory does not exist; "); - if (!(new File(timelineDir)).mkdirs()) { - throw new Error("Could not create vocalizations/files"); - } - System.out.println("Created successfully.\n"); - } - - try { - String basenameFile = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "basenames.lst"; - if ((new File(basenameFile)).exists()) { - System.out.println("Loading basenames of vocalizations from '" + basenameFile + "' list..."); - bnlVocalizations = new BasenameList(basenameFile); - System.out.println("Found " + bnlVocalizations.getLength() + " vocalizations in basename list"); - } else { - String vocalWavDir = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "wav"; - System.out.println("Loading basenames of vocalizations from '" + vocalWavDir + "' directory..."); - bnlVocalizations = new BasenameList(vocalWavDir, ".wav"); - System.out.println("Found " + bnlVocalizations.getLength() + " vocalizations in " + vocalWavDir + " directory"); - } - } catch (IOException e) { - e.printStackTrace(); - } - } - - public SortedMap getDefaultProps(DatabaseLayout db) { - this.db = db; - if (props == null) { - props = new TreeMap(); - props.put(WAVEDIR, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "wav"); - props.put(UNITFILE, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "files" + File.separator - + "vocalization_units" + db.getProp(db.MARYEXT)); - props.put(POLYORDER, "3"); - props.put(ISEXTERNALF0, "true"); - props.put(EXTERNALF0FORMAT, "sptk"); - props.put(EXTERNALEXT, ".lf0"); - props.put(PITCHDIR, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "lf0"); - props.put(SKIPSIZE, "0.005"); - props.put(WINDOWSIZE, "0.005"); - props.put(F0TIMELINE, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "files" + File.separator - + "vocalization_intonation" + db.getProp(db.MARYEXT)); - props.put(F0FEATDEF, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "features" + File.separator - + "vocalization_f0_feature_definition.txt"); - } - return props; - } - - protected void setupHelp() { - props2Help = new TreeMap(); - - } - - /** - * Reads and concatenates a list of waveforms into one single timeline file. - * - * @throws IOException - */ - @Override - public boolean compute() throws IOException, MaryConfigurationException { - - listenerUnits = new VocalizationUnitFileReader(getProp(UNITFILE)); - - // write features into timeline file - DataOutputStream out = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(new File(getProp(F0TIMELINE))))); - writeHeaderTo(out); - writeUnitFeaturesTo(out); - out.close(); - - VocalizationIntonationReader tester = new VocalizationIntonationReader(getProp(F0TIMELINE)); - int unitsOnDisk = tester.getNumberOfUnits(); - if (unitsOnDisk == listenerUnits.getNumberOfUnits()) { - System.out.println("Can read right number of units"); - return true; - } else { - System.out.println("Read wrong number of units: " + unitsOnDisk); - return false; - } - } - - /** - * - * @param out - * @throws IOException - */ - protected void writeUnitFeaturesTo(DataOutput out) throws IOException { - - int numUnits = listenerUnits.getNumberOfUnits(); - float windowSize = new Float(getProp(WINDOWSIZE)).floatValue(); - float skipSize = new Float(getProp(SKIPSIZE)).floatValue(); - - out.writeFloat(windowSize); - out.writeFloat(skipSize); - out.writeInt(numUnits); - - for (int i = 0; i < bnlVocalizations.getLength(); i++) { - - double[] f0Array = null; - - try { - f0Array = getVocalizationF0(bnlVocalizations.getName(i), false); - } catch (UnsupportedAudioFileException e) { - e.printStackTrace(); - } - - // write coeffs followed by its order - double[] coeffs = getPolynomialCoeffs(f0Array); - if (coeffs == null) { - out.writeInt(0); - } else { - out.writeInt(coeffs.length); - for (int j = 0; j < coeffs.length; j++) { - out.writeFloat((float) coeffs[j]); - } - } - - // write f0 Array followed by f0 contour array size - if (f0Array == null) { - out.writeInt(0); - } else { - out.writeInt(f0Array.length); - for (int j = 0; j < f0Array.length; j++) { - out.writeFloat((float) f0Array[j]); - } - } - } - } - - /** - * get f0 contour of vocalization f0 - * - * @param baseName - * @return - * @throws UnsupportedAudioFileException - * @throws IOException - */ - private double[] getVocalizationF0(String baseName, boolean doInterpolate) throws UnsupportedAudioFileException, IOException { - - double[] f0Array = null; - - if ("true".equals(getProp(ISEXTERNALF0))) { - - String externalFormat = getProp(EXTERNALF0FORMAT); - String externalExt = getProp(EXTERNALEXT); - System.out.println("Loading f0 contour from file : " + getProp(PITCHDIR) + File.separator + baseName + externalExt); - if ("sptk".equals(externalFormat)) { - String fileName = getProp(PITCHDIR) + File.separator + baseName + externalExt; - SPTKPitchReaderWriter sprw = new SPTKPitchReaderWriter(fileName); - f0Array = sprw.getF0Contour(); - } else if ("ptc".equals(externalFormat)) { - String fileName = getProp(PITCHDIR) + File.separator + baseName + externalExt; - PitchReaderWriter sprw = new PitchReaderWriter(fileName); - f0Array = sprw.contour; - } - } else { - PitchFileHeader params = new PitchFileHeader(); - F0TrackerAutocorrelationHeuristic tracker = new F0TrackerAutocorrelationHeuristic(params); - String waveFile = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "wav" + baseName + db.getProp(db.WAVEXT); - System.out.println("Computing f0 contour from wave file: " + waveFile); - AudioInputStream inputAudio = AudioSystem.getAudioInputStream(new File(waveFile)); - - // Enforce PCM_SIGNED encoding - if (!inputAudio.getFormat().getEncoding().equals(AudioFormat.Encoding.PCM_SIGNED)) { - inputAudio = AudioSystem.getAudioInputStream(AudioFormat.Encoding.PCM_SIGNED, inputAudio); - } - - int audioSampleRate = (int) inputAudio.getFormat().getSampleRate(); - AudioDoubleDataSource signal = new AudioDoubleDataSource(inputAudio); - double[] sentenceAudio = signal.getAllData(); - tracker.pitchAnalyze(new BufferedDoubleDataSource(sentenceAudio)); - // double frameShiftTime = tracker.getSkipSizeInSeconds(); - f0Array = tracker.getF0Contour(); - } - - if (doInterpolate) { - return interpolateF0Array(f0Array); - } - - return f0Array; - } - - /** - * to get polynomial coeffs of f0 contour - * - * @param f0Array - * @return - */ - private double[] getPolynomialCoeffs(double[] f0Array) { - - if (f0Array == null) { - return null; - } - - f0Array = cutStartEndUnvoicedSegments(f0Array); - double[] f0AndInterpolate = interpolateF0Array(f0Array); - int polynomialOrder = (new Integer(getProp(POLYORDER))).intValue(); - double[] coeffs = Polynomial.fitPolynomial(f0AndInterpolate, polynomialOrder); - return coeffs; - } - - /** - * to interpolate F0 contour values - * - * @param f0Array - * @return - */ - private double[] interpolateF0Array(double[] f0Array) { - - if (f0Array == null) { - return null; - } - - for (int j = 0; j < f0Array.length; j++) { - if (f0Array[j] == 0) { - f0Array[j] = Double.NaN; - } - } - if (f0Array.length >= 3) { - f0Array = SignalProcUtils.medianFilter(f0Array, 5); - } - - double[] f0AndInterpol; - double[] interpol = new double[f0Array.length]; - Arrays.fill(interpol, Double.NaN); - f0AndInterpol = new double[f0Array.length]; - int iLastValid = -1; - for (int j = 0; j < f0Array.length; j++) { - if (!Double.isNaN(f0Array[j])) { // a valid value - if (iLastValid == j - 1) { - // no need to interpolate - f0AndInterpol[j] = f0Array[j]; - } else { - // need to interpolate - double prevF0; - if (iLastValid < 0) { // we don't have a previous value -- use current one - prevF0 = f0Array[j]; - } else { - prevF0 = f0Array[iLastValid]; - } - double delta = (f0Array[j] - prevF0) / (j - iLastValid); - double f0 = prevF0; - for (int k = iLastValid + 1; k < j; k++) { - f0 += delta; - interpol[k] = f0; - f0AndInterpol[k] = f0; - } - } - iLastValid = j; - } - } - - double[] f0AndInterpolate = combineF0andInterpolate(f0Array, interpol); - return f0AndInterpolate; - } - - /** - * cut begin-end unvoiced segments - * - * @param array - * @return - */ - private double[] cutStartEndUnvoicedSegments(double[] array) { - - if (array == null) - return null; - - int startIndex = 0; - int endIndex = array.length; - - // find start index - for (int i = 0; i < array.length; i++) { - if (array[i] != 0) { - startIndex = i; - break; - } - } - - // find end index - for (int i = (array.length - 1); i > 0; i--) { - if (array[i] != 0) { - endIndex = i; - break; - } - } - - int newArraySize = endIndex - startIndex; - - double[] newArray = new double[newArraySize]; - System.arraycopy(array, startIndex, newArray, 0, newArraySize); - - /* - * for ( int i=0; i. - * - */ - -package marytts.tools.voiceimport.vocalizations; - -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.io.PrintWriter; -import java.util.SortedMap; -import java.util.TreeMap; - -import marytts.tools.voiceimport.DatabaseLayout; -import marytts.tools.voiceimport.PraatPitchmarker; -import marytts.util.io.BasenameList; - -public class VocalizationPitchmarker extends PraatPitchmarker { - - String vocalizationsDir; - BasenameList bnlVocalizations; - - public String getName() { - return "VocalizationPitchmarker"; - } - - @Override - protected void initialiseComp() { - tmpScript = db.getProp(db.TEMPDIR) + "script.praat"; - - if (!(new File(getProp(PRAATPMDIR))).exists()) { - - System.out.println("vocalizations/pm directory does not exist; "); - if (!(new File(getProp(PRAATPMDIR))).mkdirs()) { - throw new Error("Could not create vocalizations/pm"); - } - System.out.println("Created successfully.\n"); - - } - - try { - String basenameFile = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "basenames.lst"; - if ((new File(basenameFile)).exists()) { - System.out.println("Loading basenames of vocalizations from '" + basenameFile + "' list..."); - bnlVocalizations = new BasenameList(basenameFile); - System.out.println("Found " + bnlVocalizations.getLength() + " vocalizations in basename list"); - } else { - String vocalWavDir = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "wav"; - System.out.println("Loading basenames of vocalizations from '" + vocalWavDir + "' directory..."); - bnlVocalizations = new BasenameList(vocalWavDir, ".wav"); - System.out.println("Found " + bnlVocalizations.getLength() + " vocalizations in " + vocalWavDir + " directory"); - } - } catch (IOException e) { - e.printStackTrace(); - } - } - - public SortedMap getDefaultProps(DatabaseLayout db) { - this.db = db; - if (props == null) { - props = new TreeMap(); - props.put(COMMAND, "praat"); - if (db.getProp(db.GENDER).equals("female")) { - props.put(MINPITCH, "100"); - props.put(MAXPITCH, "500"); - } else { - props.put(MINPITCH, "75"); - props.put(MAXPITCH, "300"); - } - props.put(WAVEDIR, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "wav"); - props.put(PRAATPMDIR, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "pm"); - // vocalizationsDir = db.getProp(db.ROOTDIR)+File.separator+"vocalizations"; - } - return props; - } - - /** - * The standard compute() method of the VoiceImportComponent interface. - */ - public boolean compute() throws IOException { - - String[] baseNameArray = bnlVocalizations.getListAsArray(); - System.out.println("Computing pitchmarks for " + baseNameArray.length + " utterances."); - - /* Ensure the existence of the target pitchmark directory */ - File dir = new File(getProp(PRAATPMDIR)); - if (!dir.exists()) { - System.out.println("Creating the directory [" + getProp(PRAATPMDIR) + "]."); - dir.mkdirs(); - } - - // script.praat is provided as template. Perhaps it could be used instead of hardcoding the following: - File script = new File(tmpScript); - - if (script.exists()) - script.delete(); - PrintWriter toScript = new PrintWriter(new FileWriter(script)); - // use Praat form to provide ARGV (NOTE: these must be explicitly given during call to Praat!) - toScript.println("form Provide arguments"); - toScript.println(" sentence wavFile input.wav"); - toScript.println(" sentence pointpFile output.PointProcess"); - toScript.println(" real minPitch 75"); - toScript.println(" real maxPitch 600"); - toScript.println("endform"); - toScript.println("Read from file... 'wavFile$'"); - // Remove DC offset, if present: - toScript.println("Subtract mean"); - // First, low-pass filter the speech signal to make it more robust against noise - // (i.e., mixed noise+periodicity regions treated more likely as periodic) - toScript.println("sound = Filter (pass Hann band)... 0 1000 100"); - // Then determine pitch curve: - toScript.println("pitch = To Pitch... 0 minPitch maxPitch"); - // Get some debug info: - toScript.println("min_f0 = Get minimum... 0 0 Hertz Parabolic"); - toScript.println("max_f0 = Get maximum... 0 0 Hertz Parabolic"); - // And convert to pitch marks: - toScript.println("plus sound"); - toScript.println("To PointProcess (cc)"); - // Fill in 100 Hz pseudo-pitchmarks in unvoiced regions: - toScript.println("Voice... 0.01 0.02000000001"); - toScript.println("Write to short text file... 'pointpFile$'"); - toScript.println("lastSlash = rindex(wavFile$, \"/\")"); - toScript.println("baseName$ = right$(wavFile$, length(wavFile$) - lastSlash) - \".wav\""); - toScript.println("printline 'baseName$' f0 range: 'min_f0:0' - 'max_f0:0' Hz"); - toScript.close(); - - System.out.println("Running Praat as: " + getProp(COMMAND) + " " + tmpScript); - for (int i = 0; i < baseNameArray.length; i++) { - percent = 100 * i / baseNameArray.length; - praatPitchmarks(baseNameArray[i]); - } - - return true; - } - - /** - * @param args - */ - public static void main(String[] args) { - // TODO Auto-generated method stub - - } - -} diff --git a/marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/VocalizationTimelineMaker.java b/marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/VocalizationTimelineMaker.java deleted file mode 100644 index 48d236103e..0000000000 --- a/marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/VocalizationTimelineMaker.java +++ /dev/null @@ -1,233 +0,0 @@ -/** - * Copyright 2010 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ - -package marytts.tools.voiceimport.vocalizations; - -import java.io.ByteArrayOutputStream; -import java.io.DataOutputStream; -import java.io.File; -import java.io.IOException; -import java.text.DecimalFormat; -import java.util.SortedMap; -import java.util.TreeMap; - -import marytts.tools.voiceimport.DatabaseLayout; -import marytts.tools.voiceimport.TimelineWriter; -import marytts.tools.voiceimport.VoiceImportComponent; -import marytts.tools.voiceimport.WavReader; -import marytts.util.data.Datagram; -import marytts.util.data.ESTTrackReader; -import marytts.util.io.BasenameList; - -public class VocalizationTimelineMaker extends VoiceImportComponent { - - String vocalizationsDir; - BasenameList bnlVocalizations; - - protected DatabaseLayout db = null; - protected int percent = 0; - public final String WAVETIMELINE = "VocalizationTimelineMaker.waveTimeline"; - public final String WAVEDIR = "VocalizationTimelineMaker.inputWaveDir"; - public final String PMARKDIR = "VocalizationTimelineMaker.pitchmarkDir"; - - public final String PMDIR = "db.pmDir"; - public final String PMEXT = "db.pmExtension"; - - public String getName() { - return "VocalizationTimelineMaker"; - } - - @Override - protected void initialiseComp() { - String timelineDir = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "files"; - if (!(new File(timelineDir)).exists()) { - - System.out.println("vocalizations/files directory does not exist; "); - if (!(new File(timelineDir)).mkdirs()) { - throw new Error("Could not create vocalizations/files"); - } - System.out.println("Created successfully.\n"); - - } - - try { - String basenameFile = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "basenames.lst"; - if ((new File(basenameFile)).exists()) { - System.out.println("Loading basenames of vocalizations from '" + basenameFile + "' list..."); - bnlVocalizations = new BasenameList(basenameFile); - System.out.println("Found " + bnlVocalizations.getLength() + " vocalizations in basename list"); - } else { - String vocalWavDir = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "wav"; - System.out.println("Loading basenames of vocalizations from '" + vocalWavDir + "' directory..."); - bnlVocalizations = new BasenameList(vocalWavDir, ".wav"); - System.out.println("Found " + bnlVocalizations.getLength() + " vocalizations in " + vocalWavDir + " directory"); - } - } catch (IOException e) { - e.printStackTrace(); - } - } - - public SortedMap getDefaultProps(DatabaseLayout db) { - this.db = db; - if (props == null) { - props = new TreeMap(); - props.put(WAVEDIR, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "wav"); - props.put(PMARKDIR, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "pm"); - props.put(WAVETIMELINE, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "files" + File.separator - + "vocalization_wave_timeline" + db.getProp(db.MARYEXT)); - // vocalizationsDir = db.getProp(db.ROOTDIR)+File.separator+"vocalizations"; - } - return props; - } - - protected void setupHelp() { - props2Help = new TreeMap(); - props2Help.put(WAVETIMELINE, "file containing all wave files. Will be created by this module"); - } - - /** - * Reads and concatenates a list of waveforms into one single timeline file. - * - */ - public boolean compute() { - System.out.println("---- Making a pitch synchronous waveform timeline\n\n"); - - /* Export the basename list into an array of strings */ - String[] baseNameArray = bnlVocalizations.getListAsArray(); - System.out.println("Processing [" + baseNameArray.length + "] utterances.\n"); - - try { - /* - * 1) Determine the reference sampling rate as being the sample rate of the first encountered wav file - */ - WavReader wav = new WavReader(getProp(WAVEDIR) + baseNameArray[0] + db.getProp(db.WAVEXT)); - int globSampleRate = wav.getSampleRate(); - System.out.println("---- Detected a global sample rate of: [" + globSampleRate + "] Hz."); - - System.out.println("---- Folding the wav files according to the pitchmarks..."); - - /* 2) Open the destination timeline file */ - - /* Make the file name */ - String waveTimelineName = getProp(WAVETIMELINE); - System.out.println("Will create the waveform timeline in file [" + waveTimelineName + "]."); - - /* Processing header: */ - String processingHeader = "\n"; - - /* Instantiate the TimelineWriter: */ - TimelineWriter waveTimeline = new TimelineWriter(waveTimelineName, processingHeader, globSampleRate, 0.1); - - /* 3) Write the datagrams and feed the index */ - - float totalDuration = 0.0f; // Accumulator for the total timeline duration - long totalTime = 0l; - int numDatagrams = 0; - - /* For each EST track file: */ - ESTTrackReader pmFile = null; - for (int i = 0; i < baseNameArray.length; i++) { - percent = 100 * i / baseNameArray.length; - - /* - open+load */ - System.out.println(baseNameArray[i]); - pmFile = new ESTTrackReader(getProp(PMARKDIR) + baseNameArray[i] + db.getProp(PMEXT)); - totalDuration += pmFile.getTimeSpan(); - wav = new WavReader(getProp(WAVEDIR) + baseNameArray[i] + db.getProp(db.WAVEXT)); - short[] wave = wav.getSamples(); - /* - Reset the frame locations in the local file */ - int frameStart = 0; - int frameEnd = 0; - int duration = 0; - long localTime = 0l; - /* - For each frame in the WAV file: */ - for (int f = 0; f < pmFile.getNumFrames(); f++) { - - /* Locate the corresponding segment in the wave file */ - frameStart = frameEnd; - frameEnd = (int) ((double) pmFile.getTime(f) * (double) (globSampleRate)); - assert frameEnd <= wave.length : "Frame ends after end of wave data: " + frameEnd + " > " + wave.length; - - duration = frameEnd - frameStart; - ByteArrayOutputStream buff = new ByteArrayOutputStream(2 * duration); - DataOutputStream subWave = new DataOutputStream(buff); - for (int k = 0; k < duration; k++) { - subWave.writeShort(wave[frameStart + k]); - } - - // Handle the case when the last pitch marks falls beyond the end of the signal - - /* Feed the datagram to the timeline */ - waveTimeline.feed(new Datagram(duration, buff.toByteArray()), globSampleRate); - totalTime += duration; - localTime += duration; - numDatagrams++; - } - // System.out.println( baseNameArray[i] + " -> pm file says [" + localTime + "] samples, wav file says ["+ - // wav.getNumSamples() + "] samples." ); - } - waveTimeline.close(); - - System.out.println("---- Done."); - - /* 7) Print some stats and close the file */ - System.out.println("---- Waveform timeline result:"); - System.out.println("Number of files scanned: " + baseNameArray.length); - System.out.println("Total speech duration: [" + totalTime + "] samples / [" - + ((float) (totalTime) / (float) (globSampleRate)) + "] seconds."); - System.out.println("(Speech duration approximated from EST Track float times: [" + totalDuration + "] seconds.)"); - System.out.println("Number of frames: [" + numDatagrams + "]."); - System.out.println("Size of the index: [" + waveTimeline.getIndex().getNumIdx() + "] (" - + (waveTimeline.getIndex().getNumIdx() * 16) + " bytes, i.e. " - + new DecimalFormat("#.##").format((double) (waveTimeline.getIndex().getNumIdx()) * 16.0 / 1048576.0) - + " megs)."); - System.out.println("---- Waveform timeline done."); - - } catch (SecurityException e) { - System.err.println("Error: you don't have write access to the target database directory."); - e.printStackTrace(); - return false; - } catch (Exception e) { - e.printStackTrace(); - System.err.println(e); - return false; - } - - return (true); - } - - /** - * Provide the progress of computation, in percent, or -1 if that feature is not implemented. - * - * @return -1 if not implemented, or an integer between 0 and 100. - */ - public int getProgress() { - return percent; - } - - /** - * @param args - */ - public static void main(String[] args) { - // TODO Auto-generated method stub - - } - -} diff --git a/marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/VocalizationUnitfileWriter.java b/marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/VocalizationUnitfileWriter.java deleted file mode 100644 index 343f39a878..0000000000 --- a/marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/VocalizationUnitfileWriter.java +++ /dev/null @@ -1,319 +0,0 @@ -/** - * Copyright 2006 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.tools.voiceimport.vocalizations; - -import java.io.BufferedOutputStream; -import java.io.BufferedReader; -import java.io.DataOutputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStreamReader; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.SortedMap; -import java.util.StringTokenizer; -import java.util.TreeMap; - -import marytts.exceptions.MaryConfigurationException; -import marytts.tools.voiceimport.DatabaseLayout; -import marytts.tools.voiceimport.VoiceImportComponent; -import marytts.util.data.ESTTrackReader; -import marytts.util.data.MaryHeader; -import marytts.util.io.BasenameList; -import marytts.vocalizations.VocalizationUnitFileReader; - -/** - * Back-channel unit writer - * - * @author sathish pammi - * - */ -public class VocalizationUnitfileWriter extends VoiceImportComponent { - - protected String vocalizationsDir; - protected BasenameList bnlVocalizations; - - // protected File maryDir; - protected String unitFileName; - protected File unitlabelDir; - protected int samplingRate; - // protected String pauseSymbol; - protected String unitlabelExt = ".lab"; - - protected DatabaseLayout db = null; - protected int percent = 0; - // protected BasenameList bachChannelList; - - public String LABELDIR = "VocalizationUnitfileWriter.backchannelLabDir"; - public String UNITFILE = "VocalizationUnitfileWriter.unitFile"; - public final String PMARKDIR = "VocalizationUnitfileWriter.pitchmarkDir"; - // public String BASELIST = "VocalizationUnitfileWriter.backchannelBaseNamesList"; - - public final String PMDIR = "db.pmDir"; - public final String PMEXT = "db.pmExtension"; - - public String getName() { - return "VocalizationUnitfileWriter"; - } - - @Override - protected void initialiseComp() { - // maryDir = new File(db.getProp(db.FILEDIR)); - - samplingRate = Integer.parseInt(db.getProp(db.SAMPLINGRATE)); - // pauseSymbol = System.getProperty("pause.symbol", "pau"); - - unitFileName = getProp(UNITFILE); - unitlabelDir = new File(getProp(LABELDIR)); - - String timelineDir = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "files"; - if (!(new File(timelineDir)).exists()) { - - System.out.println("vocalizations/files directory does not exist; "); - if (!(new File(timelineDir)).mkdirs()) { - throw new Error("Could not create vocalizations/files"); - } - System.out.println("Created successfully.\n"); - - } - - try { - String basenameFile = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "basenames.lst"; - if ((new File(basenameFile)).exists()) { - System.out.println("Loading basenames of vocalizations from '" + basenameFile + "' list..."); - bnlVocalizations = new BasenameList(basenameFile); - System.out.println("Found " + bnlVocalizations.getLength() + " vocalizations in basename list"); - } else { - String vocalWavDir = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "wav"; - System.out.println("Loading basenames of vocalizations from '" + vocalWavDir + "' directory..."); - bnlVocalizations = new BasenameList(vocalWavDir, ".wav"); - System.out.println("Found " + bnlVocalizations.getLength() + " vocalizations in " + vocalWavDir + " directory"); - } - } catch (IOException e) { - e.printStackTrace(); - } - - } - - public SortedMap getDefaultProps(DatabaseLayout db) { - this.db = db; - if (props == null) { - props = new TreeMap(); - String vocalizationsrootDir = db.getProp(db.VOCALIZATIONSDIR); - props.put(LABELDIR, vocalizationsrootDir + File.separator + "lab" + System.getProperty("file.separator")); - props.put( - UNITFILE, - vocalizationsrootDir + File.separator + "files" + File.separator + "vocalization_units" - + db.getProp(db.MARYEXT)); - props.put(PMARKDIR, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "pm"); - // props.put(BASELIST, "backchannel.lst"); - } - return props; - } - - protected void setupHelp() { - props2Help = new TreeMap(); - props2Help.put(LABELDIR, "directory containing the phone labels"); - props2Help.put(UNITFILE, "file containing all phone units. Will be created by this module"); - } - - @Override - public boolean compute() throws IOException, MaryConfigurationException { - if (!unitlabelDir.exists()) { - System.out.print(LABELDIR + " " + getProp(LABELDIR) + " does not exist; "); - throw new Error("LABELDIR not found"); - } - - System.out.println("Back channel unitfile writer started..."); - BackChannelUnits bcUnits = new BackChannelUnits(unitlabelDir.getAbsolutePath(), this.bnlVocalizations); - DataOutputStream out = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(unitFileName))); - long posNumUnits = new MaryHeader(MaryHeader.LISTENERUNITS).writeTo(out); - int numberOfBCUnits = bcUnits.getNumberOfUnits(); - out.writeInt(numberOfBCUnits); - out.writeInt(samplingRate); - long globalStart = 0l; // time, given as sample position with samplingRate - for (int i = 0; i < numberOfBCUnits; i++) { - UnitLabel[] fileLabels = bcUnits.getUnitLabels(i); - double unitTimeSpan = bcUnits.getTimeSpan(i); - int localLabLength = fileLabels.length; - out.writeInt(localLabLength); - for (int j = 0; j < localLabLength; j++) { - double startTime = fileLabels[j].startTime; - double endTime = fileLabels[j].endTime; - double duration = endTime - startTime; - long end = (long) (endTime * (double) (samplingRate)); - long start = (long) (startTime * (double) (samplingRate)); - out.writeLong(globalStart + start); - out.writeInt((int) (end - start)); - out.writeInt(fileLabels[j].unitName.toCharArray().length); - out.writeChars(fileLabels[j].unitName); - } - globalStart += ((long) ((double) (unitTimeSpan) * (double) (samplingRate))); - } - - out.close(); - VocalizationUnitFileReader tester = new VocalizationUnitFileReader(unitFileName); - int unitsOnDisk = tester.getNumberOfUnits(); - if (unitsOnDisk == numberOfBCUnits) { - System.out.println("Can read right number of units: " + unitsOnDisk); - return true; - } else { - System.out.println("Read wrong number of units: " + unitsOnDisk); - return false; - } - } - - class BackChannelUnits { - int numberOfUnits; - UnitLabel[][] unitLabels; - double[] unitTimeSpan; - - BackChannelUnits(String unitlabelDir, BasenameList basenameList) throws IOException { - this.numberOfUnits = basenameList.getLength(); - unitLabels = new UnitLabel[this.numberOfUnits][]; - unitTimeSpan = new double[this.numberOfUnits]; - for (int i = 0; i < this.numberOfUnits; i++) { - String fileName = unitlabelDir + File.separator + basenameList.getName(i) + unitlabelExt; - ESTTrackReader pmFile = new ESTTrackReader(getProp(PMARKDIR) + File.separator + basenameList.getName(i) - + db.getProp(PMEXT)); - unitLabels[i] = readLabFile(fileName); - unitTimeSpan[i] = pmFile.getTimeSpan(); - } - } - - int getNumberOfUnits() { - return this.numberOfUnits; - } - - UnitLabel[] getUnitLabels(int i) { - return this.unitLabels[i]; - } - - double getTimeSpan(int i) { - return this.unitTimeSpan[i]; - } - } - - class UnitLabel { - String unitName; - double startTime; - double endTime; - int unitIndex; - - public UnitLabel(String unitName, double startTime, double endTime, int unitIndex) { - this.unitName = unitName; - this.startTime = startTime; - this.endTime = endTime; - this.unitIndex = unitIndex; - } - } - - /** - * @param labFile - * @return - */ - private UnitLabel[] readLabFile(String labFile) throws IOException { - - ArrayList lines = new ArrayList(); - BufferedReader labels = new BufferedReader(new InputStreamReader(new FileInputStream(new File(labFile)), "UTF-8")); - String line; - - // Read Label file first - // 1. Skip label file header: - while ((line = labels.readLine()) != null) { - if (line.startsWith("#")) - break; // line starting with "#" marks end of header - } - - // 2. Put data into an ArrayList - String labelUnit = null; - double startTimeStamp = 0.0; - double endTimeStamp = 0.0; - int unitIndex = 0; - while ((line = labels.readLine()) != null) { - labelUnit = null; - if (line != null) { - List labelUnitData = getLabelUnitData(line); - if (labelUnitData == null) - continue; - labelUnit = (String) labelUnitData.get(2); - unitIndex = Integer.parseInt((String) labelUnitData.get(1)); - endTimeStamp = Double.parseDouble((String) labelUnitData.get(0)); - } - if (labelUnit == null) - break; - lines.add(labelUnit.trim() + " " + startTimeStamp + " " + endTimeStamp + " " + unitIndex); - startTimeStamp = endTimeStamp; - } - labels.close(); - - UnitLabel[] ulab = new UnitLabel[lines.size()]; - Iterator itr = lines.iterator(); - for (int i = 0; itr.hasNext(); i++) { - String element = itr.next(); - String[] wrds = element.split("\\s+"); - ulab[i] = new UnitLabel(wrds[0], (new Double(wrds[1])).doubleValue(), (new Double(wrds[2])).doubleValue(), - (new Integer(wrds[3])).intValue()); - } - return ulab; - } - - /** - * To get Label Unit DATA (time stamp, index, phone unit) - * - * @param line - * @return ArrayList contains time stamp, index and phone unit - * @throws IOException - */ - private ArrayList getLabelUnitData(String line) throws IOException { - if (line == null) - return null; - if (line.trim().equals("")) - return null; - ArrayList unitData = new ArrayList(); - StringTokenizer st = new StringTokenizer(line.trim()); - // the first token is the time - unitData.add(st.nextToken()); - // the second token is the unit index - unitData.add(st.nextToken()); - // the third token is the phone - unitData.add(st.nextToken()); - return unitData; - } - - /** - * Provide the progress of computation, in percent, or -1 if that feature is not implemented. - * - * @return -1 if not implemented, or an integer between 0 and 100. - */ - public int getProgress() { - return percent; - } - - public static void main(String[] args) throws Exception { - VocalizationUnitfileWriter ufw = new VocalizationUnitfileWriter(); - new DatabaseLayout(ufw); - ufw.compute(); - } - -} diff --git a/marytts-builder/src/main/java/marytts/vocalizations/VocalisationLabelInspector.java b/marytts-builder/src/main/java/marytts/vocalizations/VocalisationLabelInspector.java deleted file mode 100644 index c258075c31..0000000000 --- a/marytts-builder/src/main/java/marytts/vocalizations/VocalisationLabelInspector.java +++ /dev/null @@ -1,252 +0,0 @@ -/** - * Copyright 2010 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.vocalizations; - -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; - -import javax.sound.sampled.AudioFormat; -import javax.sound.sampled.AudioInputStream; -import javax.sound.sampled.AudioSystem; - -import marytts.tools.voiceimport.UnitLabel; -import marytts.util.data.audio.MaryAudioUtils; -import marytts.util.io.BasenameList; - -public class VocalisationLabelInspector { - - private String inLocation; // input directory - private String outLocation; // output directory - private AudioFormat format; - - /** - * - * @param inLocation - * @param outLocation - */ - public VocalisationLabelInspector(String inLocation, String outLocation) { - this.inLocation = inLocation; - this.outLocation = outLocation; - } - - public void process(String baseName) throws IOException { - - String labelFile = inLocation + File.separator + baseName + ".lab"; - String outlabelFile = outLocation + File.separator + baseName + ".lab"; - String waveFile = inLocation + File.separator + baseName + ".wav"; - String outwaveFile = outLocation + File.separator + baseName + ".wav"; - - // read labels - UnitLabel[] vocalLabels = UnitLabel.readLabFile(labelFile); - - // read waveform - AudioInputStream audioInputStream = null; - try { - audioInputStream = AudioSystem.getAudioInputStream(new File(waveFile)); - } catch (Exception e) { - e.printStackTrace(); - } - format = audioInputStream.getFormat(); - double[] signal = MaryAudioUtils.getSamplesAsDoubleArray(audioInputStream); - - /* - * // get start and end pause durations from labels double sPauseDuration = getStartPauseDuration(vocalLabels); double - * ePauseDuration = getEndPauseDuration(vocalLabels); - * - * try { int startStampIndex = (int) (sPauseDuration * format.getSampleRate()); int endStampIndex = signal.length - (int) - * (ePauseDuration * format.getSampleRate()) - 1; double[] newSignal = new double[endStampIndex-startStampIndex]; - * System.arraycopy(signal, startStampIndex, newSignal, 0, endStampIndex-startStampIndex); - * MaryAudioUtils.writeWavFile(newSignal, outwaveFile, format); } catch(Exception e){ - * System.out.println("problem : "+baseName); } - */ - - // get start and end pause durations from labels - double sPauseStamp = getStartTimeStamp(vocalLabels); - double ePauseStamp = getEndTimeStamp(vocalLabels); - - int startStampIndex = (int) (sPauseStamp * format.getSampleRate()); - int endStampIndex = (int) (ePauseStamp * format.getSampleRate()); - double[] newSignal = new double[endStampIndex - startStampIndex]; - System.arraycopy(signal, startStampIndex, newSignal, 0, endStampIndex - startStampIndex); - MaryAudioUtils.writeWavFile(newSignal, outwaveFile, format); - UnitLabel[] newVocalLabels = removePausesFromLabels(vocalLabels); - UnitLabel.writeLabFile(newVocalLabels, outlabelFile); - - } - - private UnitLabel[] removePausesFromLabels(UnitLabel[] vocalLabels) { - - ArrayList uLabels = new ArrayList(); - - // put into a list - for (int i = 0; i < vocalLabels.length; i++) { - uLabels.add(vocalLabels[i]); - } - - // remove back pauses first - for (int i = (uLabels.size() - 1); i > 0; i--) { - if ("_".equals(vocalLabels[i].unitName)) { - uLabels.remove(i); - } else { - break; - } - } - - // remove front pauses next - for (int i = 0; i < uLabels.size(); i++) { - if ("_".equals(vocalLabels[i].unitName)) { - uLabels.remove(i); - } else { - break; - } - } - - // post process unit labels - double removedPauseTime = (uLabels.get(0)).getStartTime(); - for (int i = 0; i < uLabels.size(); i++) { - - UnitLabel ulab = uLabels.get(i); - uLabels.remove(i); - double sTime = ulab.getStartTime(); - double eTime = ulab.getEndTime(); - ulab.setStartTime(sTime - removedPauseTime); - ulab.setEndTime(eTime - removedPauseTime); - uLabels.add(i, ulab); - } - - return uLabels.toArray(new UnitLabel[0]); - } - - /** - * - * @param vocalLabels - * @return - */ - private double getStartPauseDuration(UnitLabel[] vocalLabels) { - - boolean isStartPause = false; - for (int i = 0; i < vocalLabels.length; i++) { - - if (i == 0 && "_".equals(vocalLabels[i].unitName)) { - isStartPause = true; - continue; - } - - if (isStartPause && !"_".equals(vocalLabels[i].unitName)) { - return vocalLabels[i].startTime; - } - } - - return 0.0; - } - - /** - * - * @param vocalLabels - * @return - */ - private double getEndPauseDuration(UnitLabel[] vocalLabels) { - - boolean isEndPause = false; - - for (int i = (vocalLabels.length - 1); i > 0; i--) { - - if (i == (vocalLabels.length - 1) && "_".equals(vocalLabels[i].unitName)) { - isEndPause = true; - continue; - } - - if (isEndPause && !"_".equals(vocalLabels[i].unitName)) { - return (vocalLabels[vocalLabels.length - 1].endTime - vocalLabels[i].endTime); - } - } - - return 0.0; - } - - /** - * - * @param vocalLabels - * @return - */ - private double getStartTimeStamp(UnitLabel[] vocalLabels) { - - boolean isStartPause = false; - for (int i = 0; i < vocalLabels.length; i++) { - - if (i == 0 && "_".equals(vocalLabels[i].unitName)) { - isStartPause = true; - continue; - } - - if (isStartPause && !"_".equals(vocalLabels[i].unitName)) { - return vocalLabels[i].startTime; - } - } - - return 0.0; - } - - /** - * - * @param vocalLabels - * @return - */ - private double getEndTimeStamp(UnitLabel[] vocalLabels) { - - boolean isEndPause = false; - - for (int i = (vocalLabels.length - 1); i > 0; i--) { - - if (i == (vocalLabels.length - 1) && "_".equals(vocalLabels[i].unitName)) { - isEndPause = true; - continue; - } - - if (isEndPause && !"_".equals(vocalLabels[i].unitName)) { - return vocalLabels[i].endTime; - } - } - - return vocalLabels[vocalLabels.length - 1].endTime; - } - - /** - * @param args - * @throws IOException - */ - public static void main(String[] args) throws IOException { - - // String inDirName = "/home/sathish/phd/data/original-lab-wav-sync"; - // String outDirName = "/home/sathish/phd/data/pauseless-lab-wav-sync"; - String inDirName = "/home/sathish/phd/data/original_stimulus_sync"; - String outDirName = "/home/sathish/phd/data/pauseless_stimulus_sync"; - BasenameList bnl = new BasenameList(inDirName, ".wav"); - - VocalisationLabelInspector vli = new VocalisationLabelInspector(inDirName, outDirName); - - for (int i = 0; i < bnl.getLength(); i++) { - vli.process(bnl.getName(i)); - } - - } - -} diff --git a/marytts-builder/src/main/java/marytts/vocalizations/VocalizationAnnotationReader.java b/marytts-builder/src/main/java/marytts/vocalizations/VocalizationAnnotationReader.java deleted file mode 100644 index fa5a2e847d..0000000000 --- a/marytts-builder/src/main/java/marytts/vocalizations/VocalizationAnnotationReader.java +++ /dev/null @@ -1,128 +0,0 @@ -/** - * Copyright 2010 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.vocalizations; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileReader; -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Map; - -import marytts.util.io.BasenameList; - -/** - * Reader class for manual annotation of vocalizations - * - * The format of the file should be as following examples: (first line indicates list of feature names) - * - * categories|name|voicequality|angry|sadness|amusement|happiness|contempt|solidarity|antagonism|certain|agreeing|interested| - * anticipation Obadiah_049|right|modal|2|3|1|1|2|2|1|0|0|-1|0 Poppy2_078|yeah|modal|1|1|1|2|1|2|1|-1|0|-1|0 - * Prudence_058|(laughter)|breathy|1|1|3|2|1|2|1|0|0|0|-1 Poppy2_085|yeah|breathy|1|2|1|1|1|2|1|0|0|-1|0 - * - * - * @author sathish - * - */ -public class VocalizationAnnotationReader { - - private ArrayList featureCategories; // feature categories - private Map> annotationData; // basename --> (feature category, feature value) - private BasenameList bnl = null; - - public VocalizationAnnotationReader(String featureFile) throws IOException { - this(featureFile, null); - } - - public VocalizationAnnotationReader(String featureFile, BasenameList bnl) throws IOException { - this.bnl = bnl; - formatChecker(featureFile); - featureCategories = new ArrayList(); - annotationData = new HashMap>(); - getAnnotations(featureFile); - } - - /** - * - * @param featureFile - * @throws IOException - */ - private void formatChecker(String featureFile) throws IOException { - BufferedReader bfrMean = new BufferedReader(new FileReader(new File(featureFile))); - String lineMeaning = bfrMean.readLine(); - String[] mlines = lineMeaning.split("\\|"); - int noOfStrings = mlines.length; - - while ((lineMeaning = bfrMean.readLine()) != null) { - mlines = lineMeaning.split("\\|"); - if (noOfStrings != mlines.length) { - throw new RuntimeException("the format of the file is not good."); - } - } - - System.out.println("The format of the file is good"); - bfrMean.close(); - } - - private void getAnnotations(String featureFile) throws IOException { - - BufferedReader bfrMean = new BufferedReader(new FileReader(new File(featureFile))); - String lineMeaning = bfrMean.readLine(); - String[] mlines = lineMeaning.split("\\|"); - - // read feature categories - for (int i = 1; i < mlines.length; i++) { - featureCategories.add(mlines[i].trim()); - } - - while ((lineMeaning = bfrMean.readLine()) != null) { - mlines = lineMeaning.split("\\|"); - if (bnl != null) { - if (!bnl.contains(mlines[0].trim())) { - continue; - } - } - Map featureMap = new HashMap(); - // read feature categories - for (int i = 1; i < mlines.length; i++) { - featureMap.put(featureCategories.get(i - 1), mlines[i].trim()); - } - annotationData.put(mlines[0].trim(), featureMap); - } - bfrMean.close(); - } - - public Map> getVocalizationsAnnotation() { - return this.annotationData; - } - - public ArrayList getFeatureList() { - return this.featureCategories; - } - - /** - * @param args - */ - public static void main(String[] args) { - - } - -} From 108f6a4f7f8d8fad6c59d5edffaa2aa8048c6208 Mon Sep 17 00:00:00 2001 From: Tristan Hamilton Date: Thu, 5 Feb 2015 15:13:05 +0100 Subject: [PATCH 13/31] more updated references to Target --- .../main/java/marytts/tools/dbselection/FeatureMaker.java | 2 +- .../test/java/marytts/language/de/FeatureProcessorIT.java | 2 +- .../src/test/java/marytts/features/FeatureProcessorIT.java | 5 ++--- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/marytts-builder/src/main/java/marytts/tools/dbselection/FeatureMaker.java b/marytts-builder/src/main/java/marytts/tools/dbselection/FeatureMaker.java index e1bc18da5c..2c2a0d6a0c 100644 --- a/marytts-builder/src/main/java/marytts/tools/dbselection/FeatureMaker.java +++ b/marytts-builder/src/main/java/marytts/tools/dbselection/FeatureMaker.java @@ -46,7 +46,7 @@ import marytts.modules.TargetFeatureLister; import marytts.server.Mary; import marytts.server.Request; -import marytts.unitselection.select.Target; +import marytts.features.Target; import marytts.util.MaryUtils; import marytts.util.dom.MaryDomUtils; diff --git a/marytts-languages/marytts-lang-de/src/test/java/marytts/language/de/FeatureProcessorIT.java b/marytts-languages/marytts-lang-de/src/test/java/marytts/language/de/FeatureProcessorIT.java index 6132961219..86cb7fecf9 100644 --- a/marytts-languages/marytts-lang-de/src/test/java/marytts/language/de/FeatureProcessorIT.java +++ b/marytts-languages/marytts-lang-de/src/test/java/marytts/language/de/FeatureProcessorIT.java @@ -29,7 +29,7 @@ import marytts.features.FeatureProcessorManager; import marytts.features.FeatureRegistry; import marytts.server.Mary; -import marytts.unitselection.select.Target; +import marytts.features.Target; import marytts.util.dom.MaryDomUtils; import org.junit.BeforeClass; diff --git a/marytts-languages/marytts-lang-en/src/test/java/marytts/features/FeatureProcessorIT.java b/marytts-languages/marytts-lang-en/src/test/java/marytts/features/FeatureProcessorIT.java index 87801a80cf..a221ae8392 100644 --- a/marytts-languages/marytts-lang-en/src/test/java/marytts/features/FeatureProcessorIT.java +++ b/marytts-languages/marytts-lang-en/src/test/java/marytts/features/FeatureProcessorIT.java @@ -34,9 +34,8 @@ import marytts.datatypes.MaryDataType; import marytts.datatypes.MaryXML; import marytts.features.MaryGenericFeatureProcessors.TargetElementNavigator; -import marytts.server.MaryProperties; -import marytts.unitselection.select.HalfPhoneTarget; -import marytts.unitselection.select.Target; +import marytts.features.HalfPhoneTarget; +import marytts.features.Target; import marytts.util.dom.MaryDomUtils; import org.apache.log4j.BasicConfigurator; From 5cef770dbb9ed5805248d5937c76791f79e190cc Mon Sep 17 00:00:00 2001 From: Tristan Hamilton Date: Thu, 5 Feb 2015 17:31:48 +0100 Subject: [PATCH 14/31] interpolation packages sent to purgatory --- .../unitselection/interpolation/InterpolatingSynthesizer.java | 0 .../marytts/unitselection/interpolation/InterpolatingVoice.java | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename {marytts-unitselection => marytts-purgatory}/src/main/java/marytts/unitselection/interpolation/InterpolatingSynthesizer.java (100%) rename {marytts-unitselection => marytts-purgatory}/src/main/java/marytts/unitselection/interpolation/InterpolatingVoice.java (100%) diff --git a/marytts-unitselection/src/main/java/marytts/unitselection/interpolation/InterpolatingSynthesizer.java b/marytts-purgatory/src/main/java/marytts/unitselection/interpolation/InterpolatingSynthesizer.java similarity index 100% rename from marytts-unitselection/src/main/java/marytts/unitselection/interpolation/InterpolatingSynthesizer.java rename to marytts-purgatory/src/main/java/marytts/unitselection/interpolation/InterpolatingSynthesizer.java diff --git a/marytts-unitselection/src/main/java/marytts/unitselection/interpolation/InterpolatingVoice.java b/marytts-purgatory/src/main/java/marytts/unitselection/interpolation/InterpolatingVoice.java similarity index 100% rename from marytts-unitselection/src/main/java/marytts/unitselection/interpolation/InterpolatingVoice.java rename to marytts-purgatory/src/main/java/marytts/unitselection/interpolation/InterpolatingVoice.java From 8d8803c8eff981fc2aea04fcdbcedb0cbd7ab254 Mon Sep 17 00:00:00 2001 From: Tristan Hamilton Date: Thu, 5 Feb 2015 17:40:27 +0100 Subject: [PATCH 15/31] clean code of interpolation references --- .../java/marytts/modules/synthesis/Voice.java | 23 ------------------- .../main/java/marytts/server/MaryServer.java | 5 +--- .../server/http/BaseHttpRequestHandler.java | 1 - .../java/marytts/util/MaryRuntimeUtils.java | 5 +--- 4 files changed, 2 insertions(+), 32 deletions(-) diff --git a/marytts-runtime/src/main/java/marytts/modules/synthesis/Voice.java b/marytts-runtime/src/main/java/marytts/modules/synthesis/Voice.java index d1f95ec4cb..37b26c1d73 100644 --- a/marytts-runtime/src/main/java/marytts/modules/synthesis/Voice.java +++ b/marytts-runtime/src/main/java/marytts/modules/synthesis/Voice.java @@ -70,8 +70,6 @@ import marytts.modules.phonemiser.AllophoneSet; import marytts.server.MaryProperties; import marytts.unitselection.UnitSelectionVoice; -import marytts.unitselection.interpolation.InterpolatingSynthesizer; -import marytts.unitselection.interpolation.InterpolatingVoice; import marytts.util.MaryRuntimeUtils; import marytts.util.MaryUtils; @@ -670,27 +668,6 @@ public static Voice getVoice(String name) { if (v.hasName(name)) return v; } - // Interpolating voices are created as needed: - if (InterpolatingVoice.isInterpolatingVoiceName(name)) { - InterpolatingSynthesizer interpolatingSynthesizer = null; - for (Iterator it = allVoices.iterator(); it.hasNext();) { - Voice v = it.next(); - if (v instanceof InterpolatingVoice) { - interpolatingSynthesizer = (InterpolatingSynthesizer) v.synthesizer(); - break; - } - } - if (interpolatingSynthesizer == null) - return null; - try { - Voice v = new InterpolatingVoice(interpolatingSynthesizer, name); - registerVoice(v); - return v; - } catch (Exception e) { - logger.warn("Could not create Interpolating voice:", e); - return null; - } - } return null; // no such voice found } diff --git a/marytts-runtime/src/main/java/marytts/server/MaryServer.java b/marytts-runtime/src/main/java/marytts/server/MaryServer.java index e58d9a1feb..196df602b1 100644 --- a/marytts-runtime/src/main/java/marytts/server/MaryServer.java +++ b/marytts-runtime/src/main/java/marytts/server/MaryServer.java @@ -53,7 +53,6 @@ import marytts.signalproc.effects.AudioEffects; import marytts.signalproc.effects.BaseAudioEffect; import marytts.unitselection.UnitSelectionVoice; -import marytts.unitselection.interpolation.InterpolatingVoice; import marytts.util.MaryRuntimeUtils; import marytts.util.MaryUtils; import marytts.util.data.audio.MaryAudioUtils; @@ -576,9 +575,7 @@ private boolean listLocales() { private boolean listVoices() { // list all known voices for (Voice v : Voice.getAvailableVoices()) { - if (v instanceof InterpolatingVoice) { - // do not list interpolating voice - } else if (v instanceof UnitSelectionVoice) { + if (v instanceof UnitSelectionVoice) { clientOut.println(v.getName() + " " + v.getLocale() + " " + v.gender().toString() + " " + "unitselection" + " " + ((UnitSelectionVoice) v).getDomain()); } else if (v instanceof HMMVoice) { diff --git a/marytts-runtime/src/main/java/marytts/server/http/BaseHttpRequestHandler.java b/marytts-runtime/src/main/java/marytts/server/http/BaseHttpRequestHandler.java index 4e66252d4c..90e7312aaa 100644 --- a/marytts-runtime/src/main/java/marytts/server/http/BaseHttpRequestHandler.java +++ b/marytts-runtime/src/main/java/marytts/server/http/BaseHttpRequestHandler.java @@ -44,7 +44,6 @@ import marytts.signalproc.effects.AudioEffects; import marytts.signalproc.effects.BaseAudioEffect; import marytts.unitselection.UnitSelectionVoice; -import marytts.unitselection.interpolation.InterpolatingVoice; import marytts.util.MaryRuntimeUtils; import marytts.util.MaryUtils; import marytts.util.http.Address; diff --git a/marytts-runtime/src/main/java/marytts/util/MaryRuntimeUtils.java b/marytts-runtime/src/main/java/marytts/util/MaryRuntimeUtils.java index 5016dc6fe4..27898a70e3 100644 --- a/marytts-runtime/src/main/java/marytts/util/MaryRuntimeUtils.java +++ b/marytts-runtime/src/main/java/marytts/util/MaryRuntimeUtils.java @@ -55,7 +55,6 @@ import marytts.signalproc.effects.AudioEffect; import marytts.signalproc.effects.AudioEffects; import marytts.unitselection.UnitSelectionVoice; -import marytts.unitselection.interpolation.InterpolatingVoice; import marytts.util.data.audio.AudioDestination; import marytts.util.data.audio.MaryAudioUtils; import marytts.util.dom.MaryDomUtils; @@ -405,9 +404,7 @@ public static String formJSONFromVoices(Collection voices) { for (Iterator it = voices.iterator(); it.hasNext();) { JsonObject currentVoice = new JsonObject(); Voice v = (Voice) it.next(); - if (v instanceof InterpolatingVoice) { - // do not list interpolating voice - } else if (v instanceof UnitSelectionVoice) { + if (v instanceof UnitSelectionVoice) { currentVoice.addProperty("name", v.getName()); currentVoice.addProperty("locale", v.getLocale().toString()); currentVoice.addProperty("gender", v.gender().toString()); From b4761f2dfc7229449bbd71df49bf1edca9ca175f Mon Sep 17 00:00:00 2001 From: Tristan Hamilton Date: Thu, 5 Feb 2015 17:40:48 +0100 Subject: [PATCH 16/31] more cleaning of references to Target --- .../test/java/marytts/features/TargetFeatureComputerTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/marytts-runtime/src/test/java/marytts/features/TargetFeatureComputerTest.java b/marytts-runtime/src/test/java/marytts/features/TargetFeatureComputerTest.java index 597419bd9b..7e81da337a 100644 --- a/marytts-runtime/src/test/java/marytts/features/TargetFeatureComputerTest.java +++ b/marytts-runtime/src/test/java/marytts/features/TargetFeatureComputerTest.java @@ -4,7 +4,7 @@ import java.util.Locale; -import marytts.unitselection.select.Target; +import marytts.features.Target; import marytts.util.string.ByteStringTranslator; import org.junit.Before; From a57ae79f32e0a23bd6a8872922b401c148dec239 Mon Sep 17 00:00:00 2001 From: Tristan Hamilton Date: Thu, 5 Feb 2015 17:44:14 +0100 Subject: [PATCH 17/31] remove deprecated getLexicon(voice) method from FreeTTSVoices --- .../modules/synthesis/FreeTTSVoices.java | 26 ------------------- 1 file changed, 26 deletions(-) diff --git a/marytts-runtime/src/main/java/marytts/modules/synthesis/FreeTTSVoices.java b/marytts-runtime/src/main/java/marytts/modules/synthesis/FreeTTSVoices.java index 9c90dc64c0..36e6b923dd 100644 --- a/marytts-runtime/src/main/java/marytts/modules/synthesis/FreeTTSVoices.java +++ b/marytts-runtime/src/main/java/marytts/modules/synthesis/FreeTTSVoices.java @@ -28,7 +28,6 @@ import marytts.exceptions.NoSuchPropertyException; import marytts.modules.DummyFreeTTSVoice; import marytts.server.MaryProperties; -import marytts.unitselection.UnitSelectionVoice; import marytts.util.MaryUtils; import org.apache.log4j.Logger; @@ -126,31 +125,6 @@ private static DummyFreeTTSVoice createFreeTTSVoice(Voice maryVoice) { return freeTTSVoice; } - /** - * Depending on the maryVoice and its locale, associate an existing or create a new lexicon. - * - * @param maryVoice - * @return a Lexicon; if it was freshly created, it is not yet loaded. - */ - @Deprecated - private static Lexicon getLexicon(Voice maryVoice) { - if (maryVoice instanceof UnitSelectionVoice) { - return ((UnitSelectionVoice) maryVoice).getLexicon(); - } - if (maryVoice.getLocale() == null) { - return null; - } else if (maryVoice.getLocale().equals(Locale.US)) { - if (usenLexicon == null) - usenLexicon = new CMULexicon("cmudict04"); - return usenLexicon; - } else if (maryVoice.getLocale().equals(Locale.GERMAN)) { - if (deLexicon == null) - deLexicon = new GermanLexicon(); - return deLexicon; - } - return null; - } - /** * Add a given freetts voice for a given mary voice. Repeated registration of the same voice will be ignored, so it is safe to * call this several times with the same maryVoice/freettsVoice pair. This depends on the mary property From 4c8d012c3d47f07415d8e2793fbc08006ec4f871 Mon Sep 17 00:00:00 2001 From: Tristan Hamilton Date: Fri, 6 Feb 2015 14:11:53 +0100 Subject: [PATCH 18/31] BaseHttpRequestHandler had lots of unused imports - cleaned --- .../server/http/BaseHttpRequestHandler.java | 20 +------------------ 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/marytts-runtime/src/main/java/marytts/server/http/BaseHttpRequestHandler.java b/marytts-runtime/src/main/java/marytts/server/http/BaseHttpRequestHandler.java index 90e7312aaa..707d2889a8 100644 --- a/marytts-runtime/src/main/java/marytts/server/http/BaseHttpRequestHandler.java +++ b/marytts-runtime/src/main/java/marytts/server/http/BaseHttpRequestHandler.java @@ -23,31 +23,13 @@ import java.io.FileInputStream; import java.io.IOException; import java.nio.channels.FileChannel; -import java.util.Collection; import java.util.Collections; import java.util.HashMap; -import java.util.Iterator; -import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.StringTokenizer; -import java.util.Vector; - -import marytts.Version; -import marytts.config.LanguageConfig; -import marytts.config.MaryConfig; -import marytts.datatypes.MaryDataType; -import marytts.htsengine.HMMVoice; -import marytts.modules.synthesis.Voice; -import marytts.server.MaryProperties; -import marytts.signalproc.effects.AudioEffect; -import marytts.signalproc.effects.AudioEffects; -import marytts.signalproc.effects.BaseAudioEffect; -import marytts.unitselection.UnitSelectionVoice; -import marytts.util.MaryRuntimeUtils; + import marytts.util.MaryUtils; import marytts.util.http.Address; -import marytts.util.string.StringUtils; import org.apache.http.Header; import org.apache.http.HttpEntityEnclosingRequest; From 8499c6350801be233cc7f088be22f3ab4964edba Mon Sep 17 00:00:00 2001 From: Tristan Hamilton Date: Fri, 6 Feb 2015 14:14:34 +0100 Subject: [PATCH 19/31] removed duplicate features package from marytts-unitselection module --- .../features/ByteValuedFeatureProcessor.java | 48 - .../features/ContinuousFeatureProcessor.java | 41 - .../java/marytts/features/DiphoneTarget.java | 71 - .../marytts/features/FeatureDefinition.java | 1730 ---------- .../features/FeatureProcessorManager.java | 344 -- .../marytts/features/FeatureRegistry.java | 241 -- .../java/marytts/features/FeatureVector.java | 326 -- .../marytts/features/HalfPhoneTarget.java | 60 - .../features/MaryFeatureProcessor.java | 40 - .../MaryGenericFeatureProcessors.java | 3055 ----------------- .../MaryLanguageFeatureProcessors.java | 468 --- .../features/ShortValuedFeatureProcessor.java | 48 - .../main/java/marytts/features/Target.java | 191 -- .../features/TargetFeatureComputer.java | 348 -- 14 files changed, 7011 deletions(-) delete mode 100644 marytts-unitselection/src/main/java/marytts/features/ByteValuedFeatureProcessor.java delete mode 100644 marytts-unitselection/src/main/java/marytts/features/ContinuousFeatureProcessor.java delete mode 100644 marytts-unitselection/src/main/java/marytts/features/DiphoneTarget.java delete mode 100644 marytts-unitselection/src/main/java/marytts/features/FeatureDefinition.java delete mode 100644 marytts-unitselection/src/main/java/marytts/features/FeatureProcessorManager.java delete mode 100644 marytts-unitselection/src/main/java/marytts/features/FeatureRegistry.java delete mode 100644 marytts-unitselection/src/main/java/marytts/features/FeatureVector.java delete mode 100644 marytts-unitselection/src/main/java/marytts/features/HalfPhoneTarget.java delete mode 100644 marytts-unitselection/src/main/java/marytts/features/MaryFeatureProcessor.java delete mode 100644 marytts-unitselection/src/main/java/marytts/features/MaryGenericFeatureProcessors.java delete mode 100644 marytts-unitselection/src/main/java/marytts/features/MaryLanguageFeatureProcessors.java delete mode 100644 marytts-unitselection/src/main/java/marytts/features/ShortValuedFeatureProcessor.java delete mode 100644 marytts-unitselection/src/main/java/marytts/features/Target.java delete mode 100644 marytts-unitselection/src/main/java/marytts/features/TargetFeatureComputer.java diff --git a/marytts-unitselection/src/main/java/marytts/features/ByteValuedFeatureProcessor.java b/marytts-unitselection/src/main/java/marytts/features/ByteValuedFeatureProcessor.java deleted file mode 100644 index 68c064e420..0000000000 --- a/marytts-unitselection/src/main/java/marytts/features/ByteValuedFeatureProcessor.java +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Portions Copyright 2006 DFKI GmbH. - * Portions Copyright 2001 Sun Microsystems, Inc. - * Portions Copyright 1999-2001 Language Technologies Institute, - * Carnegie Mellon University. - * All Rights Reserved. Use is subject to license terms. - * - * Permission is hereby granted, free of charge, to use and distribute - * this software and its documentation without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of this work, and to - * permit persons to whom this work is furnished to do so, subject to - * the following conditions: - * - * 1. The code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * 2. Any modifications must be clearly marked as such. - * 3. Original authors' names are not deleted. - * 4. The authors' names are not used to endorse or promote products - * derived from this software without specific prior written - * permission. - * - * DFKI GMBH AND THE CONTRIBUTORS TO THIS WORK DISCLAIM ALL WARRANTIES WITH - * REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL DFKI GMBH NOR THE - * CONTRIBUTORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL - * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR - * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS - * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF - * THIS SOFTWARE. - */ -package marytts.features; - - -/** - * Performs a specific type of processing on an item and returns an object. - */ -public interface ByteValuedFeatureProcessor extends MaryFeatureProcessor { - /** - * List the possible values of the feature processor, as clear-text values. Byte values as returned by process() can be - * translated into their string equivalent by using the byte value as an index in the String[] returned. - * - * @return an array containing the possible return values of this feature processor, in String representation. - */ - public String[] getValues(); - - public byte process(Target target); -} diff --git a/marytts-unitselection/src/main/java/marytts/features/ContinuousFeatureProcessor.java b/marytts-unitselection/src/main/java/marytts/features/ContinuousFeatureProcessor.java deleted file mode 100644 index afdb4667d6..0000000000 --- a/marytts-unitselection/src/main/java/marytts/features/ContinuousFeatureProcessor.java +++ /dev/null @@ -1,41 +0,0 @@ -/** - * Portions Copyright 2006 DFKI GmbH. - * Portions Copyright 2001 Sun Microsystems, Inc. - * Portions Copyright 1999-2001 Language Technologies Institute, - * Carnegie Mellon University. - * All Rights Reserved. Use is subject to license terms. - * - * Permission is hereby granted, free of charge, to use and distribute - * this software and its documentation without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of this work, and to - * permit persons to whom this work is furnished to do so, subject to - * the following conditions: - * - * 1. The code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * 2. Any modifications must be clearly marked as such. - * 3. Original authors' names are not deleted. - * 4. The authors' names are not used to endorse or promote products - * derived from this software without specific prior written - * permission. - * - * DFKI GMBH AND THE CONTRIBUTORS TO THIS WORK DISCLAIM ALL WARRANTIES WITH - * REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL DFKI GMBH NOR THE - * CONTRIBUTORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL - * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR - * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS - * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF - * THIS SOFTWARE. - */ -package marytts.features; - - -/** - * Performs a specific type of processing on an item and returns an object. - */ -public interface ContinuousFeatureProcessor extends MaryFeatureProcessor { - - public float process(Target target); -} diff --git a/marytts-unitselection/src/main/java/marytts/features/DiphoneTarget.java b/marytts-unitselection/src/main/java/marytts/features/DiphoneTarget.java deleted file mode 100644 index a97566e37f..0000000000 --- a/marytts-unitselection/src/main/java/marytts/features/DiphoneTarget.java +++ /dev/null @@ -1,71 +0,0 @@ -/** - * Copyright 2006 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.features; - -import marytts.features.FeatureVector; -import marytts.modules.phonemiser.Allophone; - -import org.w3c.dom.Element; - -public class DiphoneTarget extends Target { - public final HalfPhoneTarget left; - public final HalfPhoneTarget right; - - public DiphoneTarget(HalfPhoneTarget left, HalfPhoneTarget right) { - super(null, null); - this.name = left.name.substring(0, left.name.lastIndexOf("_")) + "-" - + right.name.substring(0, right.name.lastIndexOf("_")); - assert left.isRightHalf(); // the left half of this diphone must be the right half of a phone - assert right.isLeftHalf(); - this.left = left; - this.right = right; - } - - @Override - public Element getMaryxmlElement() { - throw new IllegalStateException("This method should not be called for DiphoneTargets."); - } - - public FeatureVector getFeatureVector() { - throw new IllegalStateException("This method should not be called for DiphoneTargets."); - } - - public void setFeatureVector(FeatureVector featureVector) { - throw new IllegalStateException("This method should not be called for DiphoneTargets."); - } - - public float getTargetDurationInSeconds() { - throw new IllegalStateException("This method should not be called for DiphoneTargets."); - } - - /** - * Determine whether this target is a silence target - * - * @return true if the target represents silence, false otherwise - */ - public boolean isSilence() { - throw new IllegalStateException("This method should not be called for DiphoneTargets."); - } - - public Allophone getAllophone() { - throw new IllegalStateException("This method should not be called for DiphoneTargets."); - } - -} diff --git a/marytts-unitselection/src/main/java/marytts/features/FeatureDefinition.java b/marytts-unitselection/src/main/java/marytts/features/FeatureDefinition.java deleted file mode 100644 index 00ddbaaf73..0000000000 --- a/marytts-unitselection/src/main/java/marytts/features/FeatureDefinition.java +++ /dev/null @@ -1,1730 +0,0 @@ -/** - * Copyright 2006 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.features; - -import java.io.BufferedReader; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.DataInput; -import java.io.DataInputStream; -import java.io.DataOutput; -import java.io.DataOutputStream; -import java.io.IOException; -import java.io.PrintWriter; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Set; - -import marytts.util.io.StreamUtils; -import marytts.util.string.ByteStringTranslator; -import marytts.util.string.IntStringTranslator; -import marytts.util.string.ShortStringTranslator; - -/** - * A feature definition object represents the "meaning" of feature vectors. It consists of a list of byte-valued, short-valued and - * continuous features by name and index position in the feature vector; the respective possible feature values (and corresponding - * byte and short codes); and, optionally, the weights and, for continuous features, weighting functions for each feature. - * - * @author Marc Schröder - * - */ -public class FeatureDefinition { - public static final String BYTEFEATURES = "ByteValuedFeatureProcessors"; - public static final String SHORTFEATURES = "ShortValuedFeatureProcessors"; - public static final String CONTINUOUSFEATURES = "ContinuousFeatureProcessors"; - public static final String FEATURESIMILARITY = "FeatureSimilarity"; - public static final char WEIGHT_SEPARATOR = '|'; - public static final String EDGEFEATURE = "edge"; - public static final String EDGEFEATURE_START = "start"; - public static final String EDGEFEATURE_END = "end"; - public static final String NULLVALUE = "0"; - - private int numByteFeatures; - private int numShortFeatures; - private int numContinuousFeatures; - private float[] featureWeights; - private IntStringTranslator featureNames; - // feature values: for byte and short features only - private ByteStringTranslator[] byteFeatureValues; - private ShortStringTranslator[] shortFeatureValues; - private String[] floatWeightFuncts; // for continuous features only - private float[][][] similarityMatrices = null; - - /** - * Create a feature definition object, reading textual data from the given BufferedReader. - * - * @param input - * a BufferedReader from which a textual feature definition can be read. - * @param readWeights - * a boolean indicating whether or not to read weights from input. If weights are read, they will be normalized so - * that they sum to one. - * @throws IOException - * if a reading problem occurs - * - */ - public FeatureDefinition(BufferedReader input, boolean readWeights) throws IOException { - // Section BYTEFEATURES - String line = input.readLine(); - if (line == null) - throw new IOException("Could not read from input"); - while (line.matches("^\\s*#.*") || line.matches("\\s*")) { - line = input.readLine(); - } - if (!line.trim().equals(BYTEFEATURES)) { - throw new IOException("Unexpected input: expected '" + BYTEFEATURES + "', read '" + line + "'"); - } - List byteFeatureLines = new ArrayList(); - while (true) { - line = input.readLine(); - if (line == null) - throw new IOException("Could not read from input"); - line = line.trim(); - if (line.equals(SHORTFEATURES)) - break; // Found end of section - byteFeatureLines.add(line); - } - // Section SHORTFEATURES - List shortFeatureLines = new ArrayList(); - while (true) { - line = input.readLine(); - if (line == null) - throw new IOException("Could not read from input"); - line = line.trim(); - if (line.equals(CONTINUOUSFEATURES)) - break; // Found end of section - shortFeatureLines.add(line); - } - // Section CONTINUOUSFEATURES - List continuousFeatureLines = new ArrayList(); - boolean readFeatureSimilarity = false; - while ((line = input.readLine()) != null) { // it's OK if we hit the end of the file now - line = line.trim(); - // if (line.equals(FEATURESIMILARITY) || line.equals("")) break; // Found end of section - if (line.equals(FEATURESIMILARITY)) { - // readFeatureSimilarityMatrices(input); - readFeatureSimilarity = true; - break; - } else if (line.equals("")) { // empty line: end of section - break; - } - continuousFeatureLines.add(line); - } - numByteFeatures = byteFeatureLines.size(); - numShortFeatures = shortFeatureLines.size(); - numContinuousFeatures = continuousFeatureLines.size(); - int total = numByteFeatures + numShortFeatures + numContinuousFeatures; - featureNames = new IntStringTranslator(total); - byteFeatureValues = new ByteStringTranslator[numByteFeatures]; - shortFeatureValues = new ShortStringTranslator[numShortFeatures]; - float sumOfWeights = 0; // for normalisation of weights - if (readWeights) { - featureWeights = new float[total]; - floatWeightFuncts = new String[numContinuousFeatures]; - } - - for (int i = 0; i < numByteFeatures; i++) { - line = byteFeatureLines.get(i); - String featureDef; - if (readWeights) { - int seppos = line.indexOf(WEIGHT_SEPARATOR); - if (seppos == -1) - throw new IOException("Weight separator '" + WEIGHT_SEPARATOR + "' not found in line '" + line + "'"); - String weightDef = line.substring(0, seppos).trim(); - featureDef = line.substring(seppos + 1).trim(); - // The weight definition is simply the float number: - featureWeights[i] = Float.parseFloat(weightDef); - sumOfWeights += featureWeights[i]; - if (featureWeights[i] < 0) - throw new IOException("Negative weight found in line '" + line + "'"); - } else { - featureDef = line; - } - // Now featureDef is a String in which the feature name and all feature values - // are separated by white space. - String[] nameAndValues = featureDef.split("\\s+", 2); - featureNames.set(i, nameAndValues[0]); // the feature name - byteFeatureValues[i] = new ByteStringTranslator(nameAndValues[1].split("\\s+")); // the feature values - } - - for (int i = 0; i < numShortFeatures; i++) { - line = shortFeatureLines.get(i); - String featureDef; - if (readWeights) { - int seppos = line.indexOf(WEIGHT_SEPARATOR); - if (seppos == -1) - throw new IOException("Weight separator '" + WEIGHT_SEPARATOR + "' not found in line '" + line + "'"); - String weightDef = line.substring(0, seppos).trim(); - featureDef = line.substring(seppos + 1).trim(); - // The weight definition is simply the float number: - featureWeights[numByteFeatures + i] = Float.parseFloat(weightDef); - sumOfWeights += featureWeights[numByteFeatures + i]; - if (featureWeights[numByteFeatures + i] < 0) - throw new IOException("Negative weight found in line '" + line + "'"); - } else { - featureDef = line; - } - // Now featureDef is a String in which the feature name and all feature values - // are separated by white space. - String[] nameAndValues = featureDef.split("\\s+", 2); - featureNames.set(numByteFeatures + i, nameAndValues[0]); // the feature name - shortFeatureValues[i] = new ShortStringTranslator(nameAndValues[1].split("\\s+")); // the feature values - } - - for (int i = 0; i < numContinuousFeatures; i++) { - line = continuousFeatureLines.get(i); - String featureDef; - if (readWeights) { - int seppos = line.indexOf(WEIGHT_SEPARATOR); - if (seppos == -1) - throw new IOException("Weight separator '" + WEIGHT_SEPARATOR + "' not found in line '" + line + "'"); - String weightDef = line.substring(0, seppos).trim(); - featureDef = line.substring(seppos + 1).trim(); - // The weight definition is the float number plus a definition of a weight function: - String[] weightAndFunction = weightDef.split("\\s+", 2); - featureWeights[numByteFeatures + numShortFeatures + i] = Float.parseFloat(weightAndFunction[0]); - sumOfWeights += featureWeights[numByteFeatures + numShortFeatures + i]; - if (featureWeights[numByteFeatures + numShortFeatures + i] < 0) - throw new IOException("Negative weight found in line '" + line + "'"); - try { - floatWeightFuncts[i] = weightAndFunction[1]; - } catch (ArrayIndexOutOfBoundsException e) { - // System.out.println( "weightDef string was: '" + weightDef + "'." ); - // System.out.println( "Splitting part 1: '" + weightAndFunction[0] + "'." ); - // System.out.println( "Splitting part 2: '" + weightAndFunction[1] + "'." ); - throw new RuntimeException("The string [" + weightDef + "] appears to be a badly formed" - + " weight plus weighting function definition."); - } - } else { - featureDef = line; - } - // Now featureDef is the feature name - // or the feature name followed by the word "float" - if (featureDef.endsWith("float")) { - String[] featureDefSplit = featureDef.split("\\s+", 2); - featureNames.set(numByteFeatures + numShortFeatures + i, featureDefSplit[0]); - } else { - featureNames.set(numByteFeatures + numShortFeatures + i, featureDef); - } - } - // Normalize weights to sum to one: - if (readWeights) { - for (int i = 0; i < total; i++) { - featureWeights[i] /= sumOfWeights; - } - } - - // read feature similarities here, if any - if (readFeatureSimilarity) { - readFeatureSimilarityMatrices(input); - } - } - - /** - * read similarity matrices from feature definition file - * - * @param input - * @throws IOException - */ - private void readFeatureSimilarityMatrices(BufferedReader input) throws IOException { - - String line = null; - - similarityMatrices = new float[this.getNumberOfByteFeatures()][][]; - for (int i = 0; i < this.getNumberOfByteFeatures(); i++) { - similarityMatrices[i] = null; - } - - while ((line = input.readLine()) != null) { - - if ("".equals(line)) { - return; - } - - String[] featureUniqueValues = line.trim().split("\\s+"); - String featureName = featureUniqueValues[0]; - - if (!isByteFeature(featureName)) { - throw new RuntimeException( - "Similarity matrix support is for bytefeatures only, but not for other feature types..."); - } - - int featureIndex = this.getFeatureIndex(featureName); - int noUniqValues = featureUniqueValues.length - 1; - similarityMatrices[featureIndex] = new float[noUniqValues][noUniqValues]; - - for (int i = 1; i <= noUniqValues; i++) { - - Arrays.fill(similarityMatrices[featureIndex][i - 1], 0); - String featureValue = featureUniqueValues[i]; - - String matLine = input.readLine(); - if (matLine == null) { - throw new RuntimeException("Feature definition file is having unexpected format..."); - } - - String[] lines = matLine.trim().split("\\s+"); - if (!featureValue.equals(lines[0])) { - throw new RuntimeException("Feature definition file is having unexpected format..."); - } - if (lines.length != i) { - throw new RuntimeException("Feature definition file is having unexpected format..."); - } - for (int j = 1; j < i; j++) { - float similarity = (new Float(lines[j])).floatValue(); - similarityMatrices[featureIndex][i - 1][j - 1] = similarity; - similarityMatrices[featureIndex][j - 1][i - 1] = similarity; - } - - } - } - - } - - /** - * Create a feature definition object, reading binary data from the given DataInput. - * - * @param input - * a DataInputStream or a RandomAccessFile from which a binary feature definition can be read. - * @throws IOException - * if a reading problem occurs - */ - public FeatureDefinition(DataInput input) throws IOException { - // Section BYTEFEATURES - numByteFeatures = input.readInt(); - byteFeatureValues = new ByteStringTranslator[numByteFeatures]; - // Initialise global arrays to byte feature length first; - // we have no means of knowing how many short or continuous - // features there will be, so we need to resize later. - // This will happen automatically for featureNames, but needs - // to be done by hand for featureWeights. - featureNames = new IntStringTranslator(numByteFeatures); - featureWeights = new float[numByteFeatures]; - // There is no need to normalise weights here, because - // they have already been normalized before the binary - // file was written. - for (int i = 0; i < numByteFeatures; i++) { - featureWeights[i] = input.readFloat(); - String featureName = input.readUTF(); - featureNames.set(i, featureName); - byte numberOfValuesEncoded = input.readByte(); // attention: this is an unsigned byte - int numberOfValues = numberOfValuesEncoded & 0xFF; - byteFeatureValues[i] = new ByteStringTranslator(numberOfValues); - for (int b = 0; b < numberOfValues; b++) { - String value = input.readUTF(); - byteFeatureValues[i].set((byte) b, value); - } - } - // Section SHORTFEATURES - numShortFeatures = input.readInt(); - if (numShortFeatures > 0) { - shortFeatureValues = new ShortStringTranslator[numShortFeatures]; - // resize weight array: - float[] newWeights = new float[numByteFeatures + numShortFeatures]; - System.arraycopy(featureWeights, 0, newWeights, 0, numByteFeatures); - featureWeights = newWeights; - - for (int i = 0; i < numShortFeatures; i++) { - featureWeights[numByteFeatures + i] = input.readFloat(); - String featureName = input.readUTF(); - featureNames.set(numByteFeatures + i, featureName); - short numberOfValues = input.readShort(); - shortFeatureValues[i] = new ShortStringTranslator(numberOfValues); - for (short s = 0; s < numberOfValues; s++) { - String value = input.readUTF(); - shortFeatureValues[i].set(s, value); - } - } - } - // Section CONTINUOUSFEATURES - numContinuousFeatures = input.readInt(); - floatWeightFuncts = new String[numContinuousFeatures]; - if (numContinuousFeatures > 0) { - // resize weight array: - float[] newWeights = new float[numByteFeatures + numShortFeatures + numContinuousFeatures]; - System.arraycopy(featureWeights, 0, newWeights, 0, numByteFeatures + numShortFeatures); - featureWeights = newWeights; - } - for (int i = 0; i < numContinuousFeatures; i++) { - featureWeights[numByteFeatures + numShortFeatures + i] = input.readFloat(); - floatWeightFuncts[i] = input.readUTF(); - String featureName = input.readUTF(); - featureNames.set(numByteFeatures + numShortFeatures + i, featureName); - } - } - - /** - * Create a feature definition object, reading binary data from the given byte buffer. - * - * @param input - * a byte buffer from which a binary feature definition can be read. - * @throws IOException - * if a reading problem occurs - */ - public FeatureDefinition(ByteBuffer bb) throws IOException { - // Section BYTEFEATURES - numByteFeatures = bb.getInt(); - byteFeatureValues = new ByteStringTranslator[numByteFeatures]; - // Initialise global arrays to byte feature length first; - // we have no means of knowing how many short or continuous - // features there will be, so we need to resize later. - // This will happen automatically for featureNames, but needs - // to be done by hand for featureWeights. - featureNames = new IntStringTranslator(numByteFeatures); - featureWeights = new float[numByteFeatures]; - // There is no need to normalise weights here, because - // they have already been normalized before the binary - // file was written. - for (int i = 0; i < numByteFeatures; i++) { - featureWeights[i] = bb.getFloat(); - String featureName = StreamUtils.readUTF(bb); - featureNames.set(i, featureName); - byte numberOfValuesEncoded = bb.get(); // attention: this is an unsigned byte - int numberOfValues = numberOfValuesEncoded & 0xFF; - byteFeatureValues[i] = new ByteStringTranslator(numberOfValues); - for (int b = 0; b < numberOfValues; b++) { - String value = StreamUtils.readUTF(bb); - byteFeatureValues[i].set((byte) b, value); - } - } - // Section SHORTFEATURES - numShortFeatures = bb.getInt(); - if (numShortFeatures > 0) { - shortFeatureValues = new ShortStringTranslator[numShortFeatures]; - // resize weight array: - float[] newWeights = new float[numByteFeatures + numShortFeatures]; - System.arraycopy(featureWeights, 0, newWeights, 0, numByteFeatures); - featureWeights = newWeights; - - for (int i = 0; i < numShortFeatures; i++) { - featureWeights[numByteFeatures + i] = bb.getFloat(); - String featureName = StreamUtils.readUTF(bb); - featureNames.set(numByteFeatures + i, featureName); - short numberOfValues = bb.getShort(); - shortFeatureValues[i] = new ShortStringTranslator(numberOfValues); - for (short s = 0; s < numberOfValues; s++) { - String value = StreamUtils.readUTF(bb); - shortFeatureValues[i].set(s, value); - } - } - } - // Section CONTINUOUSFEATURES - numContinuousFeatures = bb.getInt(); - floatWeightFuncts = new String[numContinuousFeatures]; - if (numContinuousFeatures > 0) { - // resize weight array: - float[] newWeights = new float[numByteFeatures + numShortFeatures + numContinuousFeatures]; - System.arraycopy(featureWeights, 0, newWeights, 0, numByteFeatures + numShortFeatures); - featureWeights = newWeights; - } - for (int i = 0; i < numContinuousFeatures; i++) { - featureWeights[numByteFeatures + numShortFeatures + i] = bb.getFloat(); - floatWeightFuncts[i] = StreamUtils.readUTF(bb); - String featureName = StreamUtils.readUTF(bb); - featureNames.set(numByteFeatures + numShortFeatures + i, featureName); - } - } - - /** - * Write this feature definition in binary format to the given output. - * - * @param out - * a DataOutputStream or RandomAccessFile to which the FeatureDefinition should be written. - * @throws IOException - * if a problem occurs while writing. - */ - public void writeBinaryTo(DataOutput out) throws IOException { - // TODO to avoid duplicate code, replace this with writeBinaryTo(out, List()) or some such - - // Section BYTEFEATURES - out.writeInt(numByteFeatures); - for (int i = 0; i < numByteFeatures; i++) { - if (featureWeights != null) { - out.writeFloat(featureWeights[i]); - } else { - out.writeFloat(0); - } - out.writeUTF(getFeatureName(i)); - - int numValues = getNumberOfValues(i); - byte numValuesEncoded = (byte) numValues; // an unsigned byte - out.writeByte(numValuesEncoded); - for (int b = 0; b < numValues; b++) { - String value = getFeatureValueAsString(i, b); - out.writeUTF(value); - } - } - // Section SHORTFEATURES - out.writeInt(numShortFeatures); - for (int i = numByteFeatures; i < numByteFeatures + numShortFeatures; i++) { - if (featureWeights != null) { - out.writeFloat(featureWeights[i]); - } else { - out.writeFloat(0); - } - out.writeUTF(getFeatureName(i)); - short numValues = (short) getNumberOfValues(i); - out.writeShort(numValues); - for (short b = 0; b < numValues; b++) { - String value = getFeatureValueAsString(i, b); - out.writeUTF(value); - } - } - // Section CONTINUOUSFEATURES - out.writeInt(numContinuousFeatures); - for (int i = numByteFeatures + numShortFeatures; i < numByteFeatures + numShortFeatures + numContinuousFeatures; i++) { - if (featureWeights != null) { - out.writeFloat(featureWeights[i]); - out.writeUTF(floatWeightFuncts[i - numByteFeatures - numShortFeatures]); - } else { - out.writeFloat(0); - out.writeUTF(""); - } - out.writeUTF(getFeatureName(i)); - } - } - - /** - * Write this feature definition in binary format to the given output, dropping featuresToDrop - * - * @param out - * a DataOutputStream or RandomAccessFile to which the FeatureDefinition should be written. - * @param featuresToDrop - * List of Integers containing the indices of features to drop from DataOutputStream - * @throws IOException - * if a problem occurs while writing. - */ - private void writeBinaryTo(DataOutput out, List featuresToDrop) throws IOException { - // how many features of each type are to be dropped - int droppedByteFeatures = 0; - int droppedShortFeatures = 0; - int droppedContinuousFeatures = 0; - for (int f : featuresToDrop) { - if (f < numByteFeatures) { - droppedByteFeatures++; - } else if (f < numByteFeatures + numShortFeatures) { - droppedShortFeatures++; - } else if (f < numByteFeatures + numShortFeatures + numContinuousFeatures) { - droppedContinuousFeatures++; - } - } - // Section BYTEFEATURES - out.writeInt(numByteFeatures - droppedByteFeatures); - for (int i = 0; i < numByteFeatures; i++) { - if (featuresToDrop.contains(i)) { - continue; - } - if (featureWeights != null) { - out.writeFloat(featureWeights[i]); - } else { - out.writeFloat(0); - } - out.writeUTF(getFeatureName(i)); - - int numValues = getNumberOfValues(i); - byte numValuesEncoded = (byte) numValues; // an unsigned byte - out.writeByte(numValuesEncoded); - for (int b = 0; b < numValues; b++) { - String value = getFeatureValueAsString(i, b); - out.writeUTF(value); - } - } - // Section SHORTFEATURES - out.writeInt(numShortFeatures - droppedShortFeatures); - for (int i = numByteFeatures; i < numByteFeatures + numShortFeatures; i++) { - if (featuresToDrop.contains(i)) { - continue; - } - if (featureWeights != null) { - out.writeFloat(featureWeights[i]); - } else { - out.writeFloat(0); - } - out.writeUTF(getFeatureName(i)); - short numValues = (short) getNumberOfValues(i); - out.writeShort(numValues); - for (short b = 0; b < numValues; b++) { - String value = getFeatureValueAsString(i, b); - out.writeUTF(value); - } - } - // Section CONTINUOUSFEATURES - out.writeInt(numContinuousFeatures - droppedContinuousFeatures); - for (int i = numByteFeatures + numShortFeatures; i < numByteFeatures + numShortFeatures + numContinuousFeatures; i++) { - if (featuresToDrop.contains(i)) { - continue; - } - if (featureWeights != null) { - out.writeFloat(featureWeights[i]); - out.writeUTF(floatWeightFuncts[i - numByteFeatures - numShortFeatures]); - } else { - out.writeFloat(0); - out.writeUTF(""); - } - out.writeUTF(getFeatureName(i)); - } - } - - /** - * Get the total number of features. - * - * @return the number of features - */ - public int getNumberOfFeatures() { - return numByteFeatures + numShortFeatures + numContinuousFeatures; - } - - /** - * Get the number of byte features. - * - * @return the number of features - */ - public int getNumberOfByteFeatures() { - return numByteFeatures; - } - - /** - * Get the number of short features. - * - * @return the number of features - */ - public int getNumberOfShortFeatures() { - return numShortFeatures; - } - - /** - * Get the number of continuous features. - * - * @return the number of features - */ - public int getNumberOfContinuousFeatures() { - return numContinuousFeatures; - } - - /** - * For the feature with the given index, return the weight. - * - * @param featureIndex - * @return a non-negative weight. - */ - public float getWeight(int featureIndex) { - return featureWeights[featureIndex]; - } - - public float[] getFeatureWeights() { - return featureWeights; - } - - /** - * Get the name of any weighting function associated with the given feature index. For byte-valued and short-valued features, - * this method will always return null; for continuous features, the method will return the name of a weighting function, or - * null. - * - * @param featureIndex - * @return the name of a weighting function, or null - */ - public String getWeightFunctionName(int featureIndex) { - return floatWeightFuncts[featureIndex - numByteFeatures - numShortFeatures]; - } - - // //////////////////// META-INFORMATION METHODS /////////////////////// - - /** - * Translate between a feature index and a feature name. - * - * @param index - * a feature index, as could be used to access a feature value in a FeatureVector. - * @return the name of the feature corresponding to the index - * @throws IndexOutOfBoundsException - * if index<0 or index>getNumberOfFeatures() - */ - public String getFeatureName(int index) { - return featureNames.get(index); - } - - /** - * Translate between an array of feature indexes and an array of feature names. - * - * @param index - * an array of feature indexes, as could be used to access a feature value in a FeatureVector. - * @return an array with the name of the features corresponding to the index - * @throws IndexOutOfBoundsException - * if any of the indexes is <0 or >getNumberOfFeatures() - */ - public String[] getFeatureNameArray(int[] index) { - String[] ret = new String[index.length]; - for (int i = 0; i < index.length; i++) { - ret[i] = getFeatureName(index[i]); - } - return (ret); - } - - /** - * Get names of all features - * - * @return an array of all feature name strings - */ - public String[] getFeatureNameArray() { - String[] names = new String[getNumberOfFeatures()]; - for (int i = 0; i < names.length; i++) { - names[i] = getFeatureName(i); - } - return (names); - } - - /** - * Get names of byte features - * - * @return an array of byte feature name strings - */ - public String[] getByteFeatureNameArray() { - String[] byteFeatureNames = new String[numByteFeatures]; - for (int i = 0; i < numByteFeatures; i++) { - assert isByteFeature(i); - byteFeatureNames[i] = getFeatureName(i); - } - return byteFeatureNames; - } - - /** - * Get names of short features - * - * @return an array of short feature name strings - */ - public String[] getShortFeatureNameArray() { - String[] shortFeatureNames = new String[numShortFeatures]; - for (int i = 0; i < numShortFeatures; i++) { - int shortFeatureIndex = numByteFeatures + i; - assert isShortFeature(shortFeatureIndex); - shortFeatureNames[i] = getFeatureName(shortFeatureIndex); - } - return shortFeatureNames; - } - - /** - * Get names of continuous features - * - * @return an array of continuous feature name strings - */ - public String[] getContinuousFeatureNameArray() { - String[] continuousFeatureNames = new String[numContinuousFeatures]; - for (int i = 0; i < numContinuousFeatures; i++) { - int continuousFeatureIndex = numByteFeatures + numShortFeatures + i; - assert isContinuousFeature(continuousFeatureIndex); - continuousFeatureNames[i] = getFeatureName(continuousFeatureIndex); - } - return continuousFeatureNames; - } - - /** - * List all feature names, separated by white space, in their order of definition. - * - * @return - */ - public String getFeatureNames() { - StringBuilder buf = new StringBuilder(); - for (int i = 0, n = getNumberOfFeatures(); i < n; i++) { - if (buf.length() > 0) - buf.append(" "); - buf.append(featureNames.get(i)); - } - return buf.toString(); - } - - /** - * Indicate whether the feature definition contains the feature with the given name - * - * @param name - * the feature name in question, e.g. "next_next_phone" - * @return - */ - public boolean hasFeature(String name) { - return featureNames.contains(name); - } - - /** - * Query a feature as identified by the given featureName as to whether the given featureValue is a known value of that - * feature. In other words, this will return true exactly if the given feature is a byte feature and - * getFeatureValueAsByte(featureName, featureValue) will not throw an exception or if the given feature is a short feature and - * getFeatureValueAsShort(featureName, featureValue) will not throw an exception. - * - * @param featureName - * @param featureValue - * @return - */ - public boolean hasFeatureValue(String featureName, String featureValue) { - return hasFeatureValue(getFeatureIndex(featureName), featureValue); - } - - /** - * Query a feature as identified by the given featureIndex as to whether the given featureValue is a known value of that - * feature. In other words, this will return true exactly if the given feature is a byte feature and - * getFeatureValueAsByte(featureIndex, featureValue) will not throw an exception or if the given feature is a short feature - * and getFeatureValueAsShort(featureIndex, featureValue) will not throw an exception. - * - * @param featureIndex - * @param featureValue - * @return - */ - public boolean hasFeatureValue(int featureIndex, String featureValue) { - if (featureIndex < 0) { - return false; - } - if (featureIndex < numByteFeatures) { - return byteFeatureValues[featureIndex].contains(featureValue); - } - if (featureIndex < numByteFeatures + numShortFeatures) { - return shortFeatureValues[featureIndex - numByteFeatures].contains(featureValue); - } - return false; - } - - /** - * Determine whether the feature with the given name is a byte feature. - * - * @param featureName - * @return true if the feature is a byte feature, false if the feature is not known or is not a byte feature - */ - public boolean isByteFeature(String featureName) { - try { - int index = getFeatureIndex(featureName); - return isByteFeature(index); - } catch (Exception e) { - return false; - } - } - - /** - * Determine whether the feature with the given index number is a byte feature. - * - * @param featureIndex - * @return true if the feature is a byte feature, false if the feature is not a byte feature or is invalid - */ - public boolean isByteFeature(int index) { - return 0 <= index && index < numByteFeatures; - } - - /** - * Determine whether the feature with the given name is a short feature. - * - * @param featureName - * @return true if the feature is a short feature, false if the feature is not known or is not a short feature - */ - public boolean isShortFeature(String featureName) { - try { - int index = getFeatureIndex(featureName); - return isShortFeature(index); - } catch (Exception e) { - return false; - } - } - - /** - * Determine whether the feature with the given index number is a short feature. - * - * @param featureIndex - * @return true if the feature is a short feature, false if the feature is not a short feature or is invalid - */ - public boolean isShortFeature(int index) { - index -= numByteFeatures; - return 0 <= index && index < numShortFeatures; - } - - /** - * Determine whether the feature with the given name is a continuous feature. - * - * @param featureName - * @return true if the feature is a continuous feature, false if the feature is not known or is not a continuous feature - */ - public boolean isContinuousFeature(String featureName) { - try { - int index = getFeatureIndex(featureName); - return isContinuousFeature(index); - } catch (Exception e) { - return false; - } - } - - /** - * Determine whether the feature with the given index number is a continuous feature. - * - * @param featureIndex - * @return true if the feature is a continuous feature, false if the feature is not a continuous feature or is invalid - */ - public boolean isContinuousFeature(int index) { - index -= numByteFeatures; - index -= numShortFeatures; - return 0 <= index && index < numContinuousFeatures; - } - - /** - * true, if given feature index contains similarity matrix - * - * @param featureIndex - * @return - */ - public boolean hasSimilarityMatrix(int featureIndex) { - - if (featureIndex >= this.getNumberOfByteFeatures()) { - return false; - } - if (this.similarityMatrices != null && this.similarityMatrices[featureIndex] != null) { - return true; - } - return false; - } - - /** - * true, if given feature name contains similarity matrix - * - * @param featureName - * @return - */ - public boolean hasSimilarityMatrix(String featureName) { - return hasSimilarityMatrix(this.getFeatureIndex(featureName)); - } - - /** - * To get a similarity between two feature values - * - * @param featureIndex - * @param i - * @param j - * @return - */ - public float getSimilarity(int featureIndex, byte i, byte j) { - if (!hasSimilarityMatrix(featureIndex)) { - throw new RuntimeException("the given feature index "); - } - return this.similarityMatrices[featureIndex][i][j]; - } - - /** - * Translate between a feature name and a feature index. - * - * @param featureName - * a valid feature name - * @return a feature index, as could be used to access a feature value in a FeatureVector. - * @throws IllegalArgumentException - * if the feature name is unknown. - */ - public int getFeatureIndex(String featureName) { - return featureNames.get(featureName); - } - - /** - * Translate between an array of feature names and an array of feature indexes. - * - * @param featureName - * an array of valid feature names - * @return an array of feature indexes, as could be used to access a feature value in a FeatureVector. - * @throws IllegalArgumentException - * if one of the feature names is unknown. - */ - public int[] getFeatureIndexArray(String[] featureName) { - int[] ret = new int[featureName.length]; - for (int i = 0; i < featureName.length; i++) { - ret[i] = getFeatureIndex(featureName[i]); - } - return (ret); - } - - /** - * Get the number of possible values for the feature with the given index number. This method must only be called for - * byte-valued or short-valued features. - * - * @param featureIndex - * the index number of the feature. - * @return for byte-valued and short-valued features, return the number of values. - * @throws IndexOutOfBoundsException - * if featureIndex < 0 or featureIndex >= getNumberOfByteFeatures() + getNumberOfShortFeatures(). - */ - public int getNumberOfValues(int featureIndex) { - if (featureIndex < numByteFeatures) - return byteFeatureValues[featureIndex].getNumberOfValues(); - featureIndex -= numByteFeatures; - if (featureIndex < numShortFeatures) - return shortFeatureValues[featureIndex].getNumberOfValues(); - throw new IndexOutOfBoundsException("Feature no. " + featureIndex + " is not a byte-valued or short-valued feature"); - } - - /** - * Get the list of possible String values for the feature with the given index number. This method must only be called for - * byte-valued or short-valued features. The position in the String array corresponds to the byte or short value of the - * feature obtained from a FeatureVector. - * - * @param featureIndex - * the index number of the feature. - * @return for byte-valued and short-valued features, return the array of String values. - * @throws IndexOutOfBoundsException - * if featureIndex < 0 or featureIndex >= getNumberOfByteFeatures() + getNumberOfShortFeatures(). - */ - public String[] getPossibleValues(int featureIndex) { - if (featureIndex < numByteFeatures) - return byteFeatureValues[featureIndex].getStringValues(); - featureIndex -= numByteFeatures; - if (featureIndex < numShortFeatures) - return shortFeatureValues[featureIndex].getStringValues(); - throw new IndexOutOfBoundsException("Feature no. " + featureIndex + " is not a byte-valued or short-valued feature"); - } - - /** - * For the feature with the given index number, translate its byte or short value to its String value. This method must only - * be called for byte-valued or short-valued features. - * - * @param featureIndex - * the index number of the feature. - * @param value - * the feature value. This must be in the range of acceptable values for the given feature. - * @return for byte-valued and short-valued features, return the String representation of the feature value. - * @throws IndexOutOfBoundsException - * if featureIndex < 0 or featureIndex >= getNumberOfByteFeatures() + getNumberOfShortFeatures() - * @throws IndexOutOfBoundsException - * if value is not a legal value for this feature - * - * - */ - public String getFeatureValueAsString(int featureIndex, int value) { - if (featureIndex < numByteFeatures) - return byteFeatureValues[featureIndex].get((byte) value); - featureIndex -= numByteFeatures; - if (featureIndex < numShortFeatures) - return shortFeatureValues[featureIndex].get((short) value); - throw new IndexOutOfBoundsException("Feature no. " + featureIndex + " is not a byte-valued or short-valued feature"); - } - - /** - * Simple access to string-based features. - * - * @param featureName - * @param fv - * @return - */ - public String getFeatureValueAsString(String featureName, FeatureVector fv) { - int i = getFeatureIndex(featureName); - return getFeatureValueAsString(i, fv.getFeatureAsInt(i)); - } - - /** - * For the feature with the given name, translate its String value to its byte value. This method must only be called for - * byte-valued features. - * - * @param featureName - * the name of the feature. - * @param value - * the feature value. This must be among the acceptable values for the given feature. - * @return for byte-valued features, return the byte representation of the feature value. - * @throws IllegalArgumentException - * if featureName is not a valid feature name, or if featureName is not a byte-valued feature. - * @throws IllegalArgumentException - * if value is not a legal value for this feature - */ - public byte getFeatureValueAsByte(String featureName, String value) { - int featureIndex = getFeatureIndex(featureName); - return getFeatureValueAsByte(featureIndex, value); - } - - /** - * For the feature with the given index number, translate its String value to its byte value. This method must only be called - * for byte-valued features. - * - * @param featureName - * the name of the feature. - * @param value - * the feature value. This must be among the acceptable values for the given feature. - * @return for byte-valued features, return the byte representation of the feature value. - * @throws IllegalArgumentException - * if featureName is not a valid feature name, or if featureName is not a byte-valued feature. - * @throws IllegalArgumentException - * if value is not a legal value for this feature - */ - public byte getFeatureValueAsByte(int featureIndex, String value) { - if (featureIndex >= numByteFeatures) - throw new IndexOutOfBoundsException("Feature no. " + featureIndex + " is not a byte-valued feature"); - try { - return byteFeatureValues[featureIndex].get(value); - } catch (IllegalArgumentException iae) { - StringBuilder message = new StringBuilder("Illegal value '" + value + "' for feature " + getFeatureName(featureIndex) - + "; Legal values are:\n"); - for (String v : getPossibleValues(featureIndex)) { - message.append(" " + v); - } - throw new IllegalArgumentException(message.toString()); - } - - } - - /** - * For the feature with the given name, translate its String value to its short value. This method must only be called for - * short-valued features. - * - * @param featureName - * the name of the feature. - * @param value - * the feature value. This must be among the acceptable values for the given feature. - * @return for short-valued features, return the short representation of the feature value. - * @throws IllegalArgumentException - * if featureName is not a valid feature name, or if featureName is not a short-valued feature. - * @throws IllegalArgumentException - * if value is not a legal value for this feature - */ - public short getFeatureValueAsShort(String featureName, String value) { - int featureIndex = getFeatureIndex(featureName); - featureIndex -= numByteFeatures; - if (featureIndex < numShortFeatures) - return shortFeatureValues[featureIndex].get(value); - throw new IndexOutOfBoundsException("Feature '" + featureName + "' is not a short-valued feature"); - } - - /** - * For the feature with the given name, translate its String value to its short value. This method must only be called for - * short-valued features. - * - * @param featureName - * the name of the feature. - * @param value - * the feature value. This must be among the acceptable values for the given feature. - * @return for short-valued features, return the short representation of the feature value. - * @throws IllegalArgumentException - * if featureName is not a valid feature name, or if featureName is not a short-valued feature. - * @throws IllegalArgumentException - * if value is not a legal value for this feature - */ - public short getFeatureValueAsShort(int featureIndex, String value) { - featureIndex -= numByteFeatures; - if (featureIndex < numShortFeatures) - return shortFeatureValues[featureIndex].get(value); - throw new IndexOutOfBoundsException("Feature no. " + featureIndex + " is not a short-valued feature"); - } - - /** - * Determine whether two feature definitions are equal, with respect to number, names, and possible values of the three kinds - * of features (byte-valued, short-valued, continuous). This method does not compare any weights. - * - * @param other - * the feature definition to compare to - * @return true if all features and values are identical, false otherwise - */ - public boolean featureEquals(FeatureDefinition other) { - if (numByteFeatures != other.numByteFeatures || numShortFeatures != other.numShortFeatures - || numContinuousFeatures != other.numContinuousFeatures) - return false; - // Compare the feature names and values for byte and short features: - for (int i = 0; i < numByteFeatures + numShortFeatures + numContinuousFeatures; i++) { - if (!getFeatureName(i).equals(other.getFeatureName(i))) - return false; - } - // Compare the values for byte and short features: - for (int i = 0; i < numByteFeatures + numShortFeatures; i++) { - if (getNumberOfValues(i) != other.getNumberOfValues(i)) - return false; - for (int v = 0, n = getNumberOfValues(i); v < n; v++) { - if (!getFeatureValueAsString(i, v).equals(other.getFeatureValueAsString(i, v))) - return false; - } - } - return true; - } - - /** - * An extension of the previous method. - */ - public String featureEqualsAnalyse(FeatureDefinition other) { - if (numByteFeatures != other.numByteFeatures) { - return ("The number of BYTE features differs: " + numByteFeatures + " versus " + other.numByteFeatures); - } - if (numShortFeatures != other.numShortFeatures) { - return ("The number of SHORT features differs: " + numShortFeatures + " versus " + other.numShortFeatures); - } - if (numContinuousFeatures != other.numContinuousFeatures) { - return ("The number of CONTINUOUS features differs: " + numContinuousFeatures + " versus " + other.numContinuousFeatures); - } - // Compare the feature names and values for byte and short features: - for (int i = 0; i < numByteFeatures + numShortFeatures + numContinuousFeatures; i++) { - if (!getFeatureName(i).equals(other.getFeatureName(i))) { - return ("The feature name differs at position [" + i + "]: " + getFeatureName(i) + " versus " + other - .getFeatureName(i)); - } - } - // Compare the values for byte and short features: - for (int i = 0; i < numByteFeatures + numShortFeatures; i++) { - if (getNumberOfValues(i) != other.getNumberOfValues(i)) { - return ("The number of values differs at position [" + i + "]: " + getNumberOfValues(i) + " versus " + other - .getNumberOfValues(i)); - } - for (int v = 0, n = getNumberOfValues(i); v < n; v++) { - if (!getFeatureValueAsString(i, v).equals(other.getFeatureValueAsString(i, v))) { - return ("The feature value differs at position [" + i + "] for feature value [" + v + "]: " - + getFeatureValueAsString(i, v) + " versus " + other.getFeatureValueAsString(i, v)); - } - } - } - return ""; - } - - /** - * Determine whether two feature definitions are equal, regarding both the actual feature definitions and the weights. The - * comparison of weights will succeed if both have no weights or if both have exactly the same weights - * - * @param other - * the feature definition to compare to - * @return true if all features, values and weights are identical, false otherwise - * @see #featureEquals(FeatureDefinition) - */ - @Override - public boolean equals(Object obj) { - if (!(obj instanceof FeatureDefinition)) - return false; - FeatureDefinition other = (FeatureDefinition) obj; - if (featureWeights == null) { - if (other.featureWeights != null) - return false; - // Both are null - } else { // featureWeights != null - if (other.featureWeights == null) - return false; - // Both != null - if (featureWeights.length != other.featureWeights.length) - return false; - for (int i = 0; i < featureWeights.length; i++) { - if (featureWeights[i] != other.featureWeights[i]) - return false; - } - assert floatWeightFuncts != null; - assert other.floatWeightFuncts != null; - if (floatWeightFuncts.length != other.floatWeightFuncts.length) - return false; - for (int i = 0; i < floatWeightFuncts.length; i++) { - if (floatWeightFuncts[i] == null) { - if (other.floatWeightFuncts[i] != null) - return false; - // Both are null - } else { // != null - if (other.floatWeightFuncts[i] == null) - return false; - // Both != null - if (!floatWeightFuncts[i].equals(other.floatWeightFuncts[i])) - return false; - } - } - } - // OK, weights are equal - return featureEquals(other); - } - - /** - * Determine whether this FeatureDefinition is a superset of, or equal to, another FeatureDefinition. - *

- * Specifically, - *

    - *
  1. every byte-valued feature in other must be in this, likewise for short-valued and continuous-valued - * features;
    2. - *
  2. for byte-valued and short-valued features, the possible feature values must be the same in this and - * other.
    3. - *
- * - * @param other - * FeatureDefinition - * @return true if - *
    - *
  1. all features in other are also in this, and every feature in other is of the same type in - * this; and
  2. every feature in other has the same possible values as the feature in this - *
    3. - *
- * false otherwise - * @author steiner - */ - public boolean contains(FeatureDefinition other) { - List thisByteFeatures = Arrays.asList(this.getByteFeatureNameArray()); - List otherByteFeatures = Arrays.asList(other.getByteFeatureNameArray()); - if (!thisByteFeatures.containsAll(otherByteFeatures)) { - return false; - } - for (String commonByteFeature : otherByteFeatures) { - String[] thisByteFeaturePossibleValues = this.getPossibleValues(this.getFeatureIndex(commonByteFeature)); - String[] otherByteFeaturePossibleValues = other.getPossibleValues(other.getFeatureIndex(commonByteFeature)); - if (!Arrays.equals(thisByteFeaturePossibleValues, otherByteFeaturePossibleValues)) { - return false; - } - } - List thisShortFeatures = Arrays.asList(this.getShortFeatureNameArray()); - List otherShortFeatures = Arrays.asList(other.getShortFeatureNameArray()); - if (!thisShortFeatures.containsAll(otherShortFeatures)) { - return false; - } - for (String commonShortFeature : otherShortFeatures) { - String[] thisShortFeaturePossibleValues = this.getPossibleValues(this.getFeatureIndex(commonShortFeature)); - String[] otherShortFeaturePossibleValues = other.getPossibleValues(other.getFeatureIndex(commonShortFeature)); - if (!Arrays.equals(thisShortFeaturePossibleValues, otherShortFeaturePossibleValues)) { - return false; - } - } - List thisContinuousFeatures = Arrays.asList(this.getContinuousFeatureNameArray()); - List otherContinuousFeatures = Arrays.asList(other.getContinuousFeatureNameArray()); - if (!thisContinuousFeatures.containsAll(otherContinuousFeatures)) { - return false; - } - return true; - } - - /** - * Create a new FeatureDefinition that contains a subset of the features in this. - * - * @param featureNamesToDrop - * array of Strings containing the names of the features to drop from the new FeatureDefinition - * @return new FeatureDefinition - * @author steiner - */ - public FeatureDefinition subset(String[] featureNamesToDrop) { - // construct a list of indices for the features to be dropped: - List featureIndicesToDrop = new ArrayList(); - for (String featureName : featureNamesToDrop) { - int featureIndex; - try { - featureIndex = getFeatureIndex(featureName); - featureIndicesToDrop.add(featureIndex); - } catch (IllegalArgumentException e) { - System.err.println("WARNING: feature " + featureName + " not found in FeatureDefinition; ignoring."); - } - } - - // create a new FeatureDefinition by way of a byte array: - FeatureDefinition subDefinition = null; - try { - ByteArrayOutputStream toMemory = new ByteArrayOutputStream(); - DataOutput output = new DataOutputStream(toMemory); - writeBinaryTo(output, featureIndicesToDrop); - - byte[] memory = toMemory.toByteArray(); - - ByteArrayInputStream fromMemory = new ByteArrayInputStream(memory); - DataInput input = new DataInputStream(fromMemory); - - subDefinition = new FeatureDefinition(input); - } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - - // make sure that subDefinition really is a subset of this - assert this.contains(subDefinition); - - return subDefinition; - } - - /** - * Create a feature vector consistent with this feature definition by reading the data from a String representation. In that - * String, the String values for each feature must be separated by white space. For example, this format is created by - * toFeatureString(FeatureVector). - * - * @param unitIndex - * an index number to assign to the feature vector - * @param featureString - * the string representation of a feature vector. - * @return the feature vector created from the String. - * @throws IllegalArgumentException - * if the feature values listed are not consistent with the feature definition. - * @see #toFeatureString(FeatureVector) - */ - public FeatureVector toFeatureVector(int unitIndex, String featureString) { - String[] featureValues = featureString.split("\\s+"); - if (featureValues.length != numByteFeatures + numShortFeatures + numContinuousFeatures) - throw new IllegalArgumentException("Expected " + (numByteFeatures + numShortFeatures + numContinuousFeatures) - + " features, got " + featureValues.length); - byte[] bytes = new byte[numByteFeatures]; - short[] shorts = new short[numShortFeatures]; - float[] floats = new float[numContinuousFeatures]; - for (int i = 0; i < numByteFeatures; i++) { - bytes[i] = Byte.parseByte(featureValues[i]); - } - for (int i = 0; i < numShortFeatures; i++) { - shorts[i] = Short.parseShort(featureValues[numByteFeatures + i]); - } - for (int i = 0; i < numContinuousFeatures; i++) { - floats[i] = Float.parseFloat(featureValues[numByteFeatures + numShortFeatures + i]); - } - return new FeatureVector(bytes, shorts, floats, unitIndex); - } - - public FeatureVector toFeatureVector(int unitIndex, byte[] bytes, short[] shorts, float[] floats) { - if (!((numByteFeatures == 0 && bytes == null || numByteFeatures == bytes.length) - && (numShortFeatures == 0 && shorts == null || numShortFeatures == shorts.length) && (numContinuousFeatures == 0 - && floats == null || numContinuousFeatures == floats.length))) { - throw new IllegalArgumentException("Expected " + numByteFeatures + " bytes (got " - + (bytes == null ? "0" : bytes.length) + "), " + numShortFeatures + " shorts (got " - + (shorts == null ? "0" : shorts.length) + "), " + numContinuousFeatures + " floats (got " - + (floats == null ? "0" : floats.length) + ")"); - } - return new FeatureVector(bytes, shorts, floats, unitIndex); - } - - /** - * Create a feature vector consistent with this feature definition by reading the data from the given input. - * - * @param input - * a DataInputStream or RandomAccessFile to read the feature values from. - * @return a FeatureVector. - */ - public FeatureVector readFeatureVector(int currentUnitIndex, DataInput input) throws IOException { - byte[] bytes = new byte[numByteFeatures]; - input.readFully(bytes); - short[] shorts = new short[numShortFeatures]; - for (int i = 0; i < shorts.length; i++) { - shorts[i] = input.readShort(); - } - float[] floats = new float[numContinuousFeatures]; - for (int i = 0; i < floats.length; i++) { - floats[i] = input.readFloat(); - } - return new FeatureVector(bytes, shorts, floats, currentUnitIndex); - } - - /** - * Create a feature vector consistent with this feature definition by reading the data from the byte buffer. - * - * @param bb - * a byte buffer to read the feature values from. - * @return a FeatureVector. - */ - public FeatureVector readFeatureVector(int currentUnitIndex, ByteBuffer bb) throws IOException { - byte[] bytes = new byte[numByteFeatures]; - bb.get(bytes); - short[] shorts = new short[numShortFeatures]; - for (int i = 0; i < shorts.length; i++) { - shorts[i] = bb.getShort(); - } - float[] floats = new float[numContinuousFeatures]; - for (int i = 0; i < floats.length; i++) { - floats[i] = bb.getFloat(); - } - return new FeatureVector(bytes, shorts, floats, currentUnitIndex); - } - - /** - * Create a feature vector that marks a start or end of a unit. All feature values are set to the neutral value "0", except - * for the EDGEFEATURE, which is set to start if start == true, to end otherwise. - * - * @param unitIndex - * index of the unit - * @param start - * true creates a start vector, false creates an end vector. - * @return a feature vector representing an edge. - */ - public FeatureVector createEdgeFeatureVector(int unitIndex, boolean start) { - int edgeFeature = getFeatureIndex(EDGEFEATURE); - assert edgeFeature < numByteFeatures; // we can assume this is byte-valued - byte edge; - if (start) - edge = getFeatureValueAsByte(edgeFeature, EDGEFEATURE_START); - else - edge = getFeatureValueAsByte(edgeFeature, EDGEFEATURE_END); - byte[] bytes = new byte[numByteFeatures]; - short[] shorts = new short[numShortFeatures]; - float[] floats = new float[numContinuousFeatures]; - for (int i = 0; i < numByteFeatures; i++) { - bytes[i] = getFeatureValueAsByte(i, NULLVALUE); - } - for (int i = 0; i < numShortFeatures; i++) { - shorts[i] = getFeatureValueAsShort(numByteFeatures + i, NULLVALUE); - } - for (int i = 0; i < numContinuousFeatures; i++) { - floats[i] = 0; - } - bytes[edgeFeature] = edge; - return new FeatureVector(bytes, shorts, floats, unitIndex); - } - - /** - * Convert a feature vector into a String representation. - * - * @param fv - * a feature vector which must be consistent with this feature definition. - * @return a String containing the String values of all features, separated by white space. - * @throws IllegalArgumentException - * if the feature vector is not consistent with this feature definition - * @throws IndexOutOfBoundsException - * if any value of the feature vector is not consistent with this feature definition - */ - public String toFeatureString(FeatureVector fv) { - if (numByteFeatures != fv.getNumberOfByteFeatures() || numShortFeatures != fv.getNumberOfShortFeatures() - || numContinuousFeatures != fv.getNumberOfContinuousFeatures()) - throw new IllegalArgumentException("Feature vector '" + fv + "' is inconsistent with feature definition"); - StringBuilder buf = new StringBuilder(); - for (int i = 0; i < numByteFeatures; i++) { - if (buf.length() > 0) - buf.append(" "); - buf.append(getFeatureValueAsString(i, fv.getByteFeature(i))); - } - for (int i = numByteFeatures; i < numByteFeatures + numShortFeatures; i++) { - if (buf.length() > 0) - buf.append(" "); - buf.append(getFeatureValueAsString(i, fv.getShortFeature(i))); - } - for (int i = numByteFeatures + numShortFeatures; i < numByteFeatures + numShortFeatures + numContinuousFeatures; i++) { - if (buf.length() > 0) - buf.append(" "); - buf.append(fv.getContinuousFeature(i)); - } - return buf.toString(); - } - - /** - * Export this feature definition in the text format which can also be read by this class. - * - * @param out - * the destination of the data - * @param writeWeights - * whether to write weights before every line - */ - public void writeTo(PrintWriter out, boolean writeWeights) { - out.println("ByteValuedFeatureProcessors"); - for (int i = 0; i < numByteFeatures; i++) { - if (writeWeights) { - out.print(featureWeights[i] + " | "); - } - out.print(getFeatureName(i)); - for (int v = 0, vmax = getNumberOfValues(i); v < vmax; v++) { - out.print(" "); - String val = getFeatureValueAsString(i, v); - out.print(val); - } - out.println(); - } - out.println("ShortValuedFeatureProcessors"); - for (int i = 0; i < numShortFeatures; i++) { - if (writeWeights) { - out.print(featureWeights[numByteFeatures + i] + " | "); - } - out.print(getFeatureName(numByteFeatures + i)); - for (int v = 0, vmax = getNumberOfValues(numByteFeatures + i); v < vmax; v++) { - out.print(" "); - String val = getFeatureValueAsString(numByteFeatures + i, v); - out.print(val); - } - out.println(); - } - out.println("ContinuousFeatureProcessors"); - for (int i = 0; i < numContinuousFeatures; i++) { - if (writeWeights) { - out.print(featureWeights[numByteFeatures + numShortFeatures + i]); - out.print(" "); - out.print(floatWeightFuncts[i]); - out.print(" | "); - } - - out.print(getFeatureName(numByteFeatures + numShortFeatures + i)); - out.println(); - } - - } - - /** - * Export this feature definition in the "all.desc" format which can be read by wagon. - * - * @param out - * the destination of the data - */ - public void generateAllDotDescForWagon(PrintWriter out) { - generateAllDotDescForWagon(out, null); - } - - /** - * Export this feature definition in the "all.desc" format which can be read by wagon. - * - * @param out - * the destination of the data - * @param featuresToIgnore - * a set of Strings containing the names of features that wagon should ignore. Can be null. - */ - public void generateAllDotDescForWagon(PrintWriter out, Set featuresToIgnore) { - out.println("("); - out.println("(occurid cluster)"); - for (int i = 0, n = getNumberOfFeatures(); i < n; i++) { - out.print("( "); - String featureName = getFeatureName(i); - out.print(featureName); - if (featuresToIgnore != null && featuresToIgnore.contains(featureName)) { - out.print(" ignore"); - } - if (i < numByteFeatures + numShortFeatures) { // list values - for (int v = 0, vmax = getNumberOfValues(i); v < vmax; v++) { - out.print(" "); - // Print values surrounded by double quotes, and make sure any - // double quotes in the value are preceded by a backslash -- - // otherwise, we get problems e.g. for sentence_punc - String val = getFeatureValueAsString(i, v); - if (val.indexOf('"') != -1) { - StringBuilder buf = new StringBuilder(); - for (int c = 0; c < val.length(); c++) { - char ch = val.charAt(c); - if (ch == '"') - buf.append("\\\""); - else - buf.append(ch); - } - val = buf.toString(); - } - out.print("\"" + val + "\""); - } - - out.println(" )"); - } else { // float feature - out.println(" float )"); - } - - } - out.println(")"); - } - - /** - * Print this feature definition plus weights to a .txt file - * - * @param out - * the destination of the data - */ - public void generateFeatureWeightsFile(PrintWriter out) { - out.println("# This file lists the features and their weights to be used for\n" + "# creating the MARY features file.\n" - + "# The same file can also be used to override weights in a run-time system.\n" - + "# Three sections are distinguished: Byte-valued, Short-valued, and\n" + "# Continuous features.\n" + "#\n" - + "# Lines starting with '#' are ignored; they can be used for comments\n" - + "# anywhere in the file. Empty lines are also ignored.\n" + "# Entries must have the following form:\n" - + "# \n" + "# | \n" + "# \n" - + "# For byte and short features, is simply the \n" - + "# (float) number representing the weight.\n" + "# For continuous features, is the\n" - + "# (float) number representing the weight, followed by an optional\n" - + "# weighting function including arguments.\n" + "#\n" - + "# The is the feature name, which in the case of\n" - + "# byte and short features is followed by the full list of feature values.\n" + "#\n" - + "# Note that the feature definitions must be identical between this file\n" - + "# and all unit feature files for individual database utterances.\n" - + "# THIS FILE WAS GENERATED AUTOMATICALLY"); - out.println(); - out.println("ByteValuedFeatureProcessors"); - List getValuesOf10 = new ArrayList(); - getValuesOf10.add("phone"); - getValuesOf10.add("ph_vc"); - getValuesOf10.add("prev_phone"); - getValuesOf10.add("next_phone"); - getValuesOf10.add("stressed"); - getValuesOf10.add("syl_break"); - getValuesOf10.add("prev_syl_break"); - getValuesOf10.add("next_is_pause"); - getValuesOf10.add("prev_is_pause"); - List getValuesOf5 = new ArrayList(); - getValuesOf5.add("cplace"); - getValuesOf5.add("ctype"); - getValuesOf5.add("cvox"); - getValuesOf5.add("vfront"); - getValuesOf5.add("vheight"); - getValuesOf5.add("vlng"); - getValuesOf5.add("vrnd"); - getValuesOf5.add("vc"); - for (int i = 0; i < numByteFeatures; i++) { - String featureName = getFeatureName(i); - if (getValuesOf10.contains(featureName)) { - out.print("10 | " + featureName); - } else { - boolean found = false; - for (String match : getValuesOf5) { - if (featureName.matches(".*" + match)) { - out.print("5 | " + featureName); - found = true; - break; - } - } - if (!found) { - out.print("0 | " + featureName); - } - } - for (int v = 0, vmax = getNumberOfValues(i); v < vmax; v++) { - String val = getFeatureValueAsString(i, v); - out.print(" " + val); - } - out.print("\n"); - } - out.println("ShortValuedFeatureProcessors"); - for (int i = numByteFeatures; i < numShortFeatures; i++) { - String featureName = getFeatureName(i); - out.print("0 | " + featureName); - for (int v = 0, vmax = getNumberOfValues(i); v < vmax; v++) { - String val = getFeatureValueAsString(i, v); - out.print(" " + val); - } - out.print("\n"); - } - out.println("ContinuousFeatureProcessors"); - for (int i = numByteFeatures; i < numByteFeatures + numContinuousFeatures; i++) { - String featureName = getFeatureName(i); - out.println("0 linear | " + featureName); - } - out.flush(); - out.close(); - } - - /** - * Compares two feature vectors in terms of how many discrete features they have in common. WARNING: this assumes that the - * feature vectors are issued from the same FeatureDefinition; only the number of features is checked for compatibility. - * - * @param v1 - * A feature vector. - * @param v2 - * Another feature vector to compare v1 with. - * @return The number of common features. - */ - public static int diff(FeatureVector v1, FeatureVector v2) { - - int ret = 0; - - /* Byte valued features */ - if (v1.byteValuedDiscreteFeatures.length < v2.byteValuedDiscreteFeatures.length) { - throw new RuntimeException("v1 and v2 don't have the same number of byte-valued features: [" - + v1.byteValuedDiscreteFeatures.length + "] versus [" + v2.byteValuedDiscreteFeatures.length + "]."); - } - for (int i = 0; i < v1.byteValuedDiscreteFeatures.length; i++) { - if (v1.byteValuedDiscreteFeatures[i] == v2.byteValuedDiscreteFeatures[i]) - ret++; - } - - /* Short valued features */ - if (v1.shortValuedDiscreteFeatures.length < v2.shortValuedDiscreteFeatures.length) { - throw new RuntimeException("v1 and v2 don't have the same number of short-valued features: [" - + v1.shortValuedDiscreteFeatures.length + "] versus [" + v2.shortValuedDiscreteFeatures.length + "]."); - } - for (int i = 0; i < v1.shortValuedDiscreteFeatures.length; i++) { - if (v1.shortValuedDiscreteFeatures[i] == v2.shortValuedDiscreteFeatures[i]) - ret++; - } - - /* TODO: would checking float-valued features make sense ? (Code below.) */ - /* float valued features */ - /* - * if ( v1.continuousFeatures.length < v2.continuousFeatures.length ) { throw new RuntimeException( - * "v1 and v2 don't have the same number of continuous features: [" + v1.continuousFeatures.length + "] versus [" + - * v2.continuousFeatures.length + "]." ); } float epsilon = 1.0e-6f; float d = 0.0f; for ( int i = 0; i < - * v1.continuousFeatures.length; i++ ) { d = ( v1.continuousFeatures[i] > v2.continuousFeatures[i] ? - * (v1.continuousFeatures[i] - v2.continuousFeatures[i]) : (v2.continuousFeatures[i] - v1.continuousFeatures[i]) ); // => - * this avoids Math.abs() if ( d < epsilon ) ret++; } - */ - - return (ret); - } - -} diff --git a/marytts-unitselection/src/main/java/marytts/features/FeatureProcessorManager.java b/marytts-unitselection/src/main/java/marytts/features/FeatureProcessorManager.java deleted file mode 100644 index 5441f80b73..0000000000 --- a/marytts-unitselection/src/main/java/marytts/features/FeatureProcessorManager.java +++ /dev/null @@ -1,344 +0,0 @@ -/** - * Copyright 2006-2007 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.features; - -import java.util.HashMap; -import java.util.Locale; -import java.util.Map; -import java.util.Set; -import java.util.TreeMap; - -import marytts.exceptions.MaryConfigurationException; -import marytts.modules.acoustic.Model; -import marytts.modules.phonemiser.AllophoneSet; -import marytts.modules.synthesis.Voice; -import marytts.server.MaryProperties; -import marytts.util.MaryRuntimeUtils; -import marytts.util.MaryUtils; - -public class FeatureProcessorManager { - protected Map processors; - - protected Map phonefeatures2values; - - protected Locale locale; - - public FeatureProcessorManager(String localeString) throws MaryConfigurationException { - this(MaryUtils.string2locale(localeString)); - } - - public FeatureProcessorManager(Locale locale) throws MaryConfigurationException { - this.locale = locale; - setupGenericFeatureProcessors(); - - AllophoneSet allophoneSet = MaryRuntimeUtils.needAllophoneSet(MaryProperties.localePrefix(locale) + ".allophoneset"); - setupPhoneFeatureProcessors(allophoneSet, null, null, null); - } - - /** - * Constructor called from a Voice that has its own acoustic models - */ - public FeatureProcessorManager(Voice voice) throws MaryConfigurationException { - this(voice.getLocale()); - registerAcousticModels(voice); - } - - /** - * Create any additional feature processors for acoustic models. - * - * @param voice - */ - protected void registerAcousticModels(Voice voice) { - Map acousticModels = voice.getAcousticModels(); - if (acousticModels == null) { - return; - } - for (Model model : acousticModels.values()) { - // does this model predict a custom continuous feature...? - String modelFeatureName = model.getFeatureName(); - if (modelFeatureName != null && !listContinuousFeatureProcessorNames().contains(modelFeatureName)) { - // ...then add a generic featureProcessor for the custom feature: - String modelAttributeName = model.getTargetAttributeName(); - MaryFeatureProcessor featureProcessor = new MaryGenericFeatureProcessors.GenericContinuousFeature( - modelFeatureName, modelAttributeName); - addFeatureProcessor(featureProcessor); - } - } - } - - /** - * This constructor should not be used anymore. It contains hard-coded feature lists. Use - * {@link #FeatureProcessorManager(Locale)} instead. - */ - @Deprecated - public FeatureProcessorManager() { - setupGenericFeatureProcessors(); - } - - @Deprecated - protected void setupHardcodedPhoneFeatureValues() { - // Set up default values for phone features: - phonefeatures2values = new HashMap(); - // cplace: 0-n/a l-labial a-alveolar p-palatal b-labio_dental d-dental v-velar g-? - phonefeatures2values.put("cplace", new String[] { "0", "l", "a", "p", "b", "d", "v", "g" }); - // ctype: 0-n/a s-stop f-fricative a-affricative n-nasal l-liquid r-r - phonefeatures2values.put("ctype", new String[] { "0", "s", "f", "a", "n", "l", "r" }); - // cvox: 0=n/a +=on -=off - phonefeatures2values.put("cvox", new String[] { "0", "+", "-" }); - // vc: 0=n/a +=vowel -=consonant - phonefeatures2values.put("vc", new String[] { "0", "+", "-" }); - // vfront: 0-n/a 1-front 2-mid 3-back - phonefeatures2values.put("vfront", new String[] { "0", "1", "2", "3" }); - // vheight: 0-n/a 1-high 2-mid 3-low - phonefeatures2values.put("vheight", new String[] { "0", "1", "2", "3" }); - // vlng: 0-n/a s-short l-long d-dipthong a-schwa - phonefeatures2values.put("vlng", new String[] { "0", "s", "l", "d", "a" }); - // vrnd: 0=n/a +=on -=off - phonefeatures2values.put("vrnd", new String[] { "0", "+", "-" }); - } - - protected void setupGenericFeatureProcessors() { - processors = new TreeMap(); - - MaryGenericFeatureProcessors.TargetElementNavigator segment = new MaryGenericFeatureProcessors.SegmentNavigator(); - MaryGenericFeatureProcessors.TargetElementNavigator prevSegment = new MaryGenericFeatureProcessors.PrevSegmentNavigator(); - MaryGenericFeatureProcessors.TargetElementNavigator nextSegment = new MaryGenericFeatureProcessors.NextSegmentNavigator(); - MaryGenericFeatureProcessors.TargetElementNavigator syllable = new MaryGenericFeatureProcessors.SyllableNavigator(); - MaryGenericFeatureProcessors.TargetElementNavigator prevSyllable = new MaryGenericFeatureProcessors.PrevSyllableNavigator(); - MaryGenericFeatureProcessors.TargetElementNavigator nextSyllable = new MaryGenericFeatureProcessors.NextSyllableNavigator(); - MaryGenericFeatureProcessors.TargetElementNavigator nextNextSyllable = new MaryGenericFeatureProcessors.NextNextSyllableNavigator(); - MaryGenericFeatureProcessors.TargetElementNavigator lastWord = new MaryGenericFeatureProcessors.LastWordInSentenceNavigator(); - - addFeatureProcessor(new MaryGenericFeatureProcessors.Edge()); - addFeatureProcessor(new MaryGenericFeatureProcessors.HalfPhoneLeftRight()); - addFeatureProcessor(new MaryGenericFeatureProcessors.Accented("accented", syllable)); - addFeatureProcessor(new MaryGenericFeatureProcessors.Stressed("stressed", syllable)); - addFeatureProcessor(new MaryGenericFeatureProcessors.Stressed("prev_stressed", prevSyllable)); - addFeatureProcessor(new MaryGenericFeatureProcessors.Stressed("next_stressed", nextSyllable)); - addFeatureProcessor(new MaryGenericFeatureProcessors.WordNumSyls()); - addFeatureProcessor(new MaryGenericFeatureProcessors.PosInSyl()); - addFeatureProcessor(new MaryGenericFeatureProcessors.SylBreak("syl_break", syllable)); - addFeatureProcessor(new MaryGenericFeatureProcessors.SylBreak("prev_syl_break", prevSyllable)); - addFeatureProcessor(new MaryGenericFeatureProcessors.PositionType()); - addFeatureProcessor(new MaryGenericFeatureProcessors.BreakIndex()); - addFeatureProcessor(new MaryGenericFeatureProcessors.IsPause("prev_is_pause", prevSegment)); - addFeatureProcessor(new MaryGenericFeatureProcessors.IsPause("next_is_pause", nextSegment)); - addFeatureProcessor(new MaryGenericFeatureProcessors.TobiAccent("tobi_accent", syllable)); - addFeatureProcessor(new MaryGenericFeatureProcessors.TobiAccent("next_tobi_accent", nextSyllable)); - addFeatureProcessor(new MaryGenericFeatureProcessors.TobiAccent("nextnext_tobi_accent", nextNextSyllable)); - addFeatureProcessor(new MaryGenericFeatureProcessors.TobiEndtone("tobi_endtone", syllable)); - addFeatureProcessor(new MaryGenericFeatureProcessors.TobiEndtone("next_tobi_endtone", nextSyllable)); - addFeatureProcessor(new MaryGenericFeatureProcessors.TobiEndtone("nextnext_tobi_endtone", nextNextSyllable)); - addFeatureProcessor(new MaryGenericFeatureProcessors.WordPunc("sentence_punc", lastWord)); - addFeatureProcessor(new MaryGenericFeatureProcessors.SylsFromPhraseStart()); - addFeatureProcessor(new MaryGenericFeatureProcessors.SylsFromPhraseEnd()); - addFeatureProcessor(new MaryGenericFeatureProcessors.StressedSylsFromPhraseStart()); - addFeatureProcessor(new MaryGenericFeatureProcessors.StressedSylsFromPhraseEnd()); - addFeatureProcessor(new MaryGenericFeatureProcessors.AccentedSylsFromPhraseStart()); - addFeatureProcessor(new MaryGenericFeatureProcessors.AccentedSylsFromPhraseEnd()); - addFeatureProcessor(new MaryGenericFeatureProcessors.SylsFromPrevStressed()); - addFeatureProcessor(new MaryGenericFeatureProcessors.SylsToNextStressed()); - addFeatureProcessor(new MaryGenericFeatureProcessors.SylsFromPrevAccent()); - addFeatureProcessor(new MaryGenericFeatureProcessors.SylsToNextAccent()); - addFeatureProcessor(new MaryGenericFeatureProcessors.WordNumSegs()); - addFeatureProcessor(new MaryGenericFeatureProcessors.SegsFromSylStart()); - addFeatureProcessor(new MaryGenericFeatureProcessors.SegsFromSylEnd()); - addFeatureProcessor(new MaryGenericFeatureProcessors.SylNumSegs()); - addFeatureProcessor(new MaryGenericFeatureProcessors.SentenceNumPhrases()); - addFeatureProcessor(new MaryGenericFeatureProcessors.SentenceNumWords()); - addFeatureProcessor(new MaryGenericFeatureProcessors.PhraseNumWords()); - addFeatureProcessor(new MaryGenericFeatureProcessors.PhraseNumSyls()); - addFeatureProcessor(new MaryGenericFeatureProcessors.SegsFromWordStart()); - addFeatureProcessor(new MaryGenericFeatureProcessors.SegsFromWordEnd()); - addFeatureProcessor(new MaryGenericFeatureProcessors.SylsFromWordStart()); - addFeatureProcessor(new MaryGenericFeatureProcessors.SylsFromWordEnd()); - addFeatureProcessor(new MaryGenericFeatureProcessors.WordsFromPhraseStart()); - addFeatureProcessor(new MaryGenericFeatureProcessors.WordsFromPhraseEnd()); - addFeatureProcessor(new MaryGenericFeatureProcessors.WordsFromSentenceStart()); - addFeatureProcessor(new MaryGenericFeatureProcessors.WordsFromSentenceEnd()); - addFeatureProcessor(new MaryGenericFeatureProcessors.PhrasesFromSentenceStart()); - addFeatureProcessor(new MaryGenericFeatureProcessors.PhrasesFromSentenceEnd()); - addFeatureProcessor(new MaryGenericFeatureProcessors.NextAccent()); - addFeatureProcessor(new MaryGenericFeatureProcessors.PrevAccent()); - addFeatureProcessor(new MaryGenericFeatureProcessors.PhraseEndtone()); - addFeatureProcessor(new MaryGenericFeatureProcessors.PrevPhraseEndtone()); - addFeatureProcessor(new MaryGenericFeatureProcessors.PrevPunctuation()); - addFeatureProcessor(new MaryGenericFeatureProcessors.NextPunctuation()); - addFeatureProcessor(new MaryGenericFeatureProcessors.WordsFromPrevPunctuation()); - addFeatureProcessor(new MaryGenericFeatureProcessors.WordsToNextPunctuation()); - addFeatureProcessor(new MaryGenericFeatureProcessors.Selection_Prosody(syllable)); - addFeatureProcessor(new MaryGenericFeatureProcessors.Style()); - - addFeatureProcessor(new MaryGenericFeatureProcessors.UnitDuration()); - addFeatureProcessor(new MaryGenericFeatureProcessors.UnitLogF0()); - addFeatureProcessor(new MaryGenericFeatureProcessors.UnitLogF0Delta()); - } - - /** - * Get the locale for this feature processor manager. Locale-specific subclasses must override this method to return the - * respective locale. This base class returns null to indicate that the feature processor manager can be used as a fallback - * for any locale. - * - * @return - */ - public Locale getLocale() { - return locale; - } - - /** - * Provide a space-separated list of the feature names for all the feature processors known to this feature processor manager. - * The list is unsorted except that byte-valued feature processors come first, followed by short-valued feature processors, - * followed by continuous feature processors. - * - * @return - */ - public String listFeatureProcessorNames() { - String bytes = listByteValuedFeatureProcessorNames(); - String shorts = listShortValuedFeatureProcessorNames(); - String conts = listContinuousFeatureProcessorNames(); - StringBuilder sb = new StringBuilder(bytes.length() + shorts.length() + conts.length() + 2); - sb.append(bytes); - if (bytes.length() > 0 && shorts.length() > 0) { - sb.append(" "); - } - sb.append(shorts); - if (conts.length() > 0) { - sb.append(" "); - } - sb.append(conts); - return sb.toString(); - } - - public String listByteValuedFeatureProcessorNames() { - StringBuilder sb = new StringBuilder(); - for (String name : processors.keySet()) { - MaryFeatureProcessor fp = processors.get(name); - if (fp instanceof ByteValuedFeatureProcessor) { - if (sb.length() > 0) - sb.append(" "); - sb.append(name); - } - } - return sb.toString(); - } - - public String listShortValuedFeatureProcessorNames() { - StringBuilder sb = new StringBuilder(); - for (String name : processors.keySet()) { - MaryFeatureProcessor fp = processors.get(name); - if (fp instanceof ShortValuedFeatureProcessor) { - if (sb.length() > 0) - sb.append(" "); - sb.append(name); - } - } - return sb.toString(); - } - - public String listContinuousFeatureProcessorNames() { - StringBuilder sb = new StringBuilder(); - for (String name : processors.keySet()) { - MaryFeatureProcessor fp = processors.get(name); - if (fp instanceof ContinuousFeatureProcessor) { - if (sb.length() > 0) - sb.append(" "); - sb.append(name); - } - } - return sb.toString(); - } - - protected void addFeatureProcessor(MaryFeatureProcessor fp) { - processors.put(fp.getName(), fp); - } - - public MaryFeatureProcessor getFeatureProcessor(String name) { - return processors.get(name); - } - - /** - * Set up phone feature processors based on phoneset. - * - * @param phoneset - * the AllophoneSet used for the current locale. - * @param phoneValues - * optional. If null, will query phoneset. - * @param pauseSymbol - * optional. If null, will query phoneset. - * @param featuresToValues - * map listing the possible values for each feature. Optional. If null, will query phoneset. - */ - protected void setupPhoneFeatureProcessors(AllophoneSet phoneset, String[] phoneValues, String pauseSymbol, - Map featuresToValues) { - MaryGenericFeatureProcessors.TargetElementNavigator segment = new MaryGenericFeatureProcessors.SegmentNavigator(); - - if (phoneValues == null) { - String[] pValues = (String[]) phoneset.getAllophoneNames().toArray(new String[0]); - phoneValues = new String[pValues.length + 1]; - phoneValues[0] = "0"; - System.arraycopy(pValues, 0, phoneValues, 1, pValues.length); - } - if (pauseSymbol == null) { - pauseSymbol = phoneset.getSilence().name(); - } - addFeatureProcessor(new MaryLanguageFeatureProcessors.Phone("phone", phoneValues, pauseSymbol, segment)); - addFeatureProcessor(new MaryLanguageFeatureProcessors.HalfPhoneUnitName(phoneValues, pauseSymbol)); - addFeatureProcessor(new MaryLanguageFeatureProcessors.SegOnsetCoda(phoneset)); - // Phone features: - Set featureNames; - if (featuresToValues != null) - featureNames = featuresToValues.keySet(); - else - featureNames = phoneset.getPhoneFeatures(); - for (String feature : featureNames) { - String[] values; - if (featuresToValues != null) - values = featuresToValues.get(feature); - else - values = phoneset.getPossibleFeatureValues(feature); - addFeatureProcessor(new MaryLanguageFeatureProcessors.PhoneFeature(phoneset, "ph_" + feature, feature, values, - pauseSymbol, segment)); - } - - Map segments = new HashMap(); - - segments.put("prev", new MaryGenericFeatureProcessors.PrevSegmentNavigator()); - segments.put("prev_prev", new MaryGenericFeatureProcessors.PrevPrevSegmentNavigator()); - segments.put("next", new MaryGenericFeatureProcessors.NextSegmentNavigator()); - segments.put("next_next", new MaryGenericFeatureProcessors.NextNextSegmentNavigator()); - - for (String position : segments.keySet()) { - MaryGenericFeatureProcessors.TargetElementNavigator navi = segments.get(position); - addFeatureProcessor(new MaryLanguageFeatureProcessors.Phone(position + "_phone", phoneValues, pauseSymbol, navi)); - // Phone features: - for (String feature : featureNames) { - String[] values; - if (featuresToValues != null) - values = featuresToValues.get(feature); - else - values = phoneset.getPossibleFeatureValues(feature); - addFeatureProcessor(new MaryLanguageFeatureProcessors.PhoneFeature(phoneset, position + "_" + feature, feature, - values, pauseSymbol, navi)); - } - } - - } - -} diff --git a/marytts-unitselection/src/main/java/marytts/features/FeatureRegistry.java b/marytts-unitselection/src/main/java/marytts/features/FeatureRegistry.java deleted file mode 100644 index 425e38f875..0000000000 --- a/marytts-unitselection/src/main/java/marytts/features/FeatureRegistry.java +++ /dev/null @@ -1,241 +0,0 @@ -/** - * Copyright 2009 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ - -package marytts.features; - -import java.util.Collection; -import java.util.Comparator; -import java.util.HashMap; -import java.util.Locale; -import java.util.Map; -import java.util.StringTokenizer; -import java.util.TreeSet; - -import marytts.modules.synthesis.Voice; - -import org.apache.commons.collections.map.MultiKeyMap; - -/** - * @author marc - * - */ -public class FeatureRegistry { - /** - * No instances of this class. - */ - private FeatureRegistry() { - } - - private static Map managersByLocale = new HashMap(); - private static Map managersByVoice = new HashMap(); - private static FeatureProcessorManager fallbackManager = null; - private static MultiKeyMap/* */computers = new MultiKeyMap(); - - /** - * Set the given feature processor manager as the one to use for the given locale. - * - * @param locale - * @param mgr - */ - public static void setFeatureProcessorManager(Locale locale, FeatureProcessorManager mgr) { - managersByLocale.put(locale, mgr); - } - - /** - * Set the given feature processor manager as the one to use when no voice- or locale-specific feature processor manager can - * be found. - * - * @param mgr - */ - public static void setFallbackFeatureProcessorManager(FeatureProcessorManager mgr) { - fallbackManager = mgr; - } - - /** - * Set the given feature processor manager as the one to use for the given voice. - * - * @param voice - * @param mgr - */ - public static void setFeatureProcessorManager(Voice voice, FeatureProcessorManager mgr) { - managersByVoice.put(voice, mgr); - } - - /** - * Get the feature processor manager associated with the given voice, if any. - * - * @param voice - * @return the feature processor manager, or null if there is no voice-specific feature processor manager. - */ - public static FeatureProcessorManager getFeatureProcessorManager(Voice voice) { - return managersByVoice.get(voice); - } - - /** - * Get the feature processor manager associated with the given locale, if any. - * - * @param locale - * @return the feature processor manager, or null if there is no locale-specific feature processor manager. - */ - public static FeatureProcessorManager getFeatureProcessorManager(Locale locale) { - FeatureProcessorManager m = managersByLocale.get(locale); - if (m != null) - return m; - // Maybe locale is language_COUNTRY, so look up by language also: - Locale lang = new Locale(locale.getLanguage()); - return managersByLocale.get(lang); - } - - /** - * Get the fallback feature processor manager which should be used if there is no voice- or locale-specific feature processor - * manager. - * - * @return - */ - public static FeatureProcessorManager getFallbackFeatureProcessorManager() { - return fallbackManager; - } - - /** - * For the given voice, return the best feature manager. That is either the voice-specific feature manager, if any, or the - * locale-specific feature manager, if any, or the language-specific feature manager, if any, or the fallback feature manager. - * - * @param voice - * @return a feature processor manager object. If this returns null, something is broken. - */ - public static FeatureProcessorManager determineBestFeatureProcessorManager(Voice voice) { - FeatureProcessorManager mgr = getFeatureProcessorManager(voice); - if (mgr == null) { - mgr = determineBestFeatureProcessorManager(voice.getLocale()); - } - return mgr; - } - - /** - * For the given locale, return the best feature manager. That is either the locale-specific feature manager, if any, or the - * language-specific feature manager, if any, or the fallback feature manager. - * - * @param locale - * @return a feature processor manager object. If this returns null, something is broken. - */ - public static FeatureProcessorManager determineBestFeatureProcessorManager(Locale locale) { - FeatureProcessorManager mgr = getFeatureProcessorManager(locale); - // Locale can have been en_US etc, i.e. language + country; let's try - // language only as well. - if (mgr == null) { - Locale lang = new Locale(locale.getLanguage()); - mgr = getFeatureProcessorManager(lang); - } - if (mgr == null) { - mgr = getFallbackFeatureProcessorManager(); - } - assert mgr != null; - return mgr; - } - - public static Collection getSupportedLocales() { - Collection locales = new TreeSet(new Comparator() { - public int compare(Locale o1, Locale o2) { - if (o1 == null) { - if (o2 == null) - return 0; - return -1; - } - if (o2 == null) { - return 1; - } - return o1.toString().compareTo(o2.toString()); - } - }); - locales.addAll(managersByLocale.keySet()); - return locales; - } - - /** - * Obtain a TargetFeatureComputer that knows how to compute features for a Target using the given set of feature processor - * names. These names must be known to the given Feature processor manager. - * - * @param mgr - * @param features - * a String containing the names of the feature processors to use, separated by white space, and in the right order - * (byte-valued discrete feature processors first, then short-valued, then continuous). If features is null, use - * all available features processors. - * @return a target feature computer - * @throws IllegalArgumentException - * if one of the features is not known to the manager - */ - public static TargetFeatureComputer getTargetFeatureComputer(FeatureProcessorManager mgr, String features) { - if (features == null) { - features = mgr.listFeatureProcessorNames(); - } else { - // verify that each feature is known to the mgr - StringTokenizer st = new StringTokenizer(features); - while (st.hasMoreTokens()) { - String feature = st.nextToken(); - if (mgr.getFeatureProcessor(feature) == null) { - throw new IllegalArgumentException("Feature processor manager '" + mgr.getClass().toString() - + "' does not know the feature '" + feature + "'"); - } - - } - } - TargetFeatureComputer tfc = (TargetFeatureComputer) computers.get(mgr, features); - if (tfc == null) { - tfc = new TargetFeatureComputer(mgr, features); - } - return tfc; - } - - /** - * Convenience method for getting a suitable target feature computer for the given locale and list of features. A feature - * processor for the given locale is looked up using {@link #getFeatureProcessorManager(Locale)} or, if that fails, using - * {@link #getFallbackFeatureProcessorManager()}. - * - * @see #getTargetFeatureComputer(FeatureProcessorManager, String) - * @param locale - * @param features - * a String containing the names of the feature processors to use, separated by white space, and in the right order - * (byte-valued discrete feature processors first, then short-valued, then continuous) - * @return a target feature computer - */ - public static TargetFeatureComputer getTargetFeatureComputer(Locale locale, String features) { - FeatureProcessorManager mgr = determineBestFeatureProcessorManager(locale); - return getTargetFeatureComputer(mgr, features); - } - - /** - * Convenience method for getting a suitable target feature computer for the given voice and list of features. A feature - * processor for the given voice is looked up using {@link #getFeatureProcessorManager(Voice)} or, if that fails, using - * {@link #getFeatureProcessorManager(Locale)} using the voice locale or, if that also fails, using - * {@link #getFallbackFeatureProcessorManager()}. - * - * @see #getTargetFeatureComputer(FeatureProcessorManager, String) - * @param locale - * @param features - * a String containing the names of the feature processors to use, separated by white space, and in the right order - * (byte-valued discrete feature processors first, then short-valued, then continuous) - * @return a target feature computer - */ - public static TargetFeatureComputer getTargetFeatureComputer(Voice voice, String features) { - FeatureProcessorManager mgr = determineBestFeatureProcessorManager(voice); - return getTargetFeatureComputer(mgr, features); - } - -} diff --git a/marytts-unitselection/src/main/java/marytts/features/FeatureVector.java b/marytts-unitselection/src/main/java/marytts/features/FeatureVector.java deleted file mode 100644 index a3e4d0c624..0000000000 --- a/marytts-unitselection/src/main/java/marytts/features/FeatureVector.java +++ /dev/null @@ -1,326 +0,0 @@ -/** - * Copyright 2000-2009 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.features; - -import java.io.DataOutput; -import java.io.IOException; - -/** - * Compact representation of a list of byte-valued, short-valued and float-valued (continuous) features. In the user interface, - * features are identified through one index number, covering all three types of features. For example, a feature vector - * consisting of three bytes, four shorts and two floats will have nine features. Use getByteFeature(), getShortFeature() and - * getContinuousFeature() to access the features as primitives; note that you have to respect the index restrictions, i.e. in the - * example, calling getShortFeature(6) would be OK, but getShortFeature(2) or getShortFeature(8) would throw an - * IndexOutOfBoundsException. Use isShortFeature(i) to test whether a given feature is a short feature. Alternatively, you can use - * an Object interface to access all features in a uniform way: getFeature(i) will return a Number object for all valid indexes. - * - * @author Marc Schröder - * - */ -public class FeatureVector { - public enum FeatureType { - byteValued, shortValued, floatValued - }; - - public final int unitIndex; - public final byte[] byteValuedDiscreteFeatures; - public final short[] shortValuedDiscreteFeatures; - public final float[] continuousFeatures; - - public FeatureVector(byte[] byteValuedDiscreteFeatures, short[] shortValuedDiscreteFeatures, float[] continuousFeatures, - int setUnitIndex) { - this.byteValuedDiscreteFeatures = byteValuedDiscreteFeatures; - this.shortValuedDiscreteFeatures = shortValuedDiscreteFeatures; - this.continuousFeatures = continuousFeatures; - if (setUnitIndex < 0) { - throw new RuntimeException("The unit index can't be negative or null when instanciating a new feature vector."); - } - this.unitIndex = setUnitIndex; - } - - /** - * Is this an edge vector? - * - * @return isEdge - */ - public boolean isEdgeVector(int edgeIndex) { - String edgeValue = getFeature(edgeIndex).toString(); - return (!edgeValue.equals(FeatureDefinition.NULLVALUE)); - } - - public FeatureType getFeatureType(int featureIndex) { - FeatureType t = null; - if (featureIndex < 0 - || featureIndex >= byteValuedDiscreteFeatures.length + shortValuedDiscreteFeatures.length - + continuousFeatures.length) { - throw new IllegalArgumentException("Index " + featureIndex + " is out of range [0, " + getLength() + "["); - } - if (featureIndex < byteValuedDiscreteFeatures.length) { - t = FeatureType.byteValued; - } else if (featureIndex < byteValuedDiscreteFeatures.length + shortValuedDiscreteFeatures.length) { - t = FeatureType.shortValued; - } else if (featureIndex < byteValuedDiscreteFeatures.length + shortValuedDiscreteFeatures.length - + continuousFeatures.length) { - t = FeatureType.floatValued; - } - return t; - } - - /** - * Get the total number of features in this feature vector. - * - * @return the number of features, irrespective of their type - */ - public int getLength() { - return byteValuedDiscreteFeatures.length + shortValuedDiscreteFeatures.length + continuousFeatures.length; - } - - /** - * Get the index of the unit to which the current feature vector applies. - * - * @return The related unit index. - */ - public int getUnitIndex() { - return (unitIndex); - } - - /** - * The number of byte features in this feature vector. - * - * @return a number of byte features, possibly 0. - */ - public int getNumberOfByteFeatures() { - return byteValuedDiscreteFeatures.length; - } - - /** - * The number of short features in this feature vector. - * - * @return a number of short features, possibly 0. - */ - public int getNumberOfShortFeatures() { - return shortValuedDiscreteFeatures.length; - } - - /** - * The number of continuous features in this feature vector. - * - * @return a number of continuous features, possibly 0. - */ - public int getNumberOfContinuousFeatures() { - return continuousFeatures.length; - } - - /** - * A uniform way to access features in this feature vector. - * - * @param index - * a feature index between 0 and getLength()-1 - * @return a Number object, which will be a Byte, a Short or a Float depending on the type of the feature with the given index - * number. - */ - public Number getFeature(int index) { - if (index < byteValuedDiscreteFeatures.length) - return new Byte(byteValuedDiscreteFeatures[index]); - index -= byteValuedDiscreteFeatures.length; - if (index < shortValuedDiscreteFeatures.length) - return new Short(shortValuedDiscreteFeatures[index]); - index -= shortValuedDiscreteFeatures.length; - if (index < continuousFeatures.length) - return new Float(continuousFeatures[index]); - throw new IndexOutOfBoundsException(); - } - - /** - * A wrapper to getFeature(), to get the result as an int value, e.g., for subsequent array indexing. - * - * @param index - * A feature index between 0 and getLength()-1. - * @return The feature value, as an int. - * - * @see FeatureVector#getFeature(int) - */ - public int getFeatureAsInt(int index) { - return (getFeature(index).intValue()); - } - - /** - * A wrapper to getFeature(), to get the result as an String value, e.g., for subsequent System.out output. - * - * @param index - * A feature index between 0 and getLength()-1. - * @param feaDef - * A FeatureDefinition object allowing to decode the feature value. - * @return The feature value, as a String. - * - * @see FeatureVector#getFeature(int) - */ - public String getFeatureAsString(int index, FeatureDefinition feaDef) { - if (index < byteValuedDiscreteFeatures.length) - return feaDef.getFeatureValueAsString(index, byteValuedDiscreteFeatures[index]); - throw new IndexOutOfBoundsException(); - } - - /** - * An efficient way to access byte-valued features in this feature vector. - * - * @param index - * the index number of the byte-valued feature in this feature vector. - * @return the byte value of the feature with the given index. - * @throws IndexOutOfBoundsException - * if index<0 or index >= getNumberOfByteFeatures(). - * @see #getNumberOfByteFeatures() - * @see #isByteFeature() - */ - public final byte getByteFeature(int index) { - if (index < 0 || index >= byteValuedDiscreteFeatures.length) { - throw new IndexOutOfBoundsException(index + " is not between 0 and " + byteValuedDiscreteFeatures.length); - } - return byteValuedDiscreteFeatures[index]; - } - - /** - * An efficient way to access short-valued features in this feature vector. - * - * @param index - * the index number of the short-valued feature in this feature vector. - * @return the short value of the feature with the given index. - * @throws IndexOutOfBoundsException - * if index= getNumberOfByteFeatures()+getNumberOfShortFeatures(). - * @see #getNumberOfByteFeatures() - * @see #getNumberOfShortFeatures() - * @see #isShortFeature() - */ - public final short getShortFeature(int index) { - return shortValuedDiscreteFeatures[index - byteValuedDiscreteFeatures.length]; - } - - /** - * An efficient way to access continuous features in this feature vector. - * - * @param index - * the index number of the continuous feature in this feature vector. - * @return the float value of the feature with the given index. - * @throws IndexOutOfBoundsException - * if index= getLength(). - * @see #getNumberOfByteFeatures() - * @see #getNumberOfShortFeatures() - * @see #getNumberOfContinuousFeatures() - * @see #getLength() - * @see #isContinuousFeature() - */ - public final float getContinuousFeature(int index) { - return continuousFeatures[index - byteValuedDiscreteFeatures.length - shortValuedDiscreteFeatures.length]; - } - - /** - * Test whether the feature with the given index number is a byte feature. - * - * @param index - * @return This will return true exactly if getByteFeature(index) would return a value without throwing an exception, i.e. if - * index>=0 and index < getNumberOfByteFeatures(). - */ - public boolean isByteFeature(int index) { - return 0 <= index && index < byteValuedDiscreteFeatures.length; - } - - /** - * Test whether the feature with the given index number is a short feature. - * - * @param index - * @return This will return true exactly if getShortFeature(index) would return a value without throwing an exception, i.e. if - * index>=getNumberOfByteFeatures() and index < getNumberOfByteFeatures()+getNumberOfShortFeatures(). - */ - public boolean isShortFeature(int index) { - return byteValuedDiscreteFeatures.length <= index - && index < byteValuedDiscreteFeatures.length + shortValuedDiscreteFeatures.length; - } - - /** - * Test whether the feature with the given index number is a continuous feature. - * - * @param index - * @return This will return true exactly if getContinuousFeature(index) would return a value without throwing an exception, - * i.e. if index>=getNumberOfByteFeatures()+getNumberOfShortFeatures() and index < getLength(). - */ - public boolean isContinuousFeature(int index) { - return byteValuedDiscreteFeatures.length + shortValuedDiscreteFeatures.length <= index - && index < byteValuedDiscreteFeatures.length + shortValuedDiscreteFeatures.length + continuousFeatures.length; - } - - public byte[] getByteValuedDiscreteFeatures() { - return byteValuedDiscreteFeatures; - } - - public short[] getShortValuedDiscreteFeatures() { - return shortValuedDiscreteFeatures; - } - - public float[] getContinuousFeatures() { - return continuousFeatures; - } - - /** - * Write a binary representation of this feature vector to the given data output. - * - * @param out - * the DataOutputStream or RandomAccessFile in which to write the binary representation of the feature vector. - * @return - */ - public void writeTo(DataOutput out) throws IOException { - if (byteValuedDiscreteFeatures != null) { - out.write(byteValuedDiscreteFeatures); - } - if (shortValuedDiscreteFeatures != null) { - for (int i = 0; i < shortValuedDiscreteFeatures.length; i++) { - out.writeShort(shortValuedDiscreteFeatures[i]); - } - } - if (continuousFeatures != null) { - for (int i = 0; i < continuousFeatures.length; i++) { - out.writeFloat(continuousFeatures[i]); - } - } - } - - /** - * Return a string representation of this set of target features; feature values separated by spaces. - */ - public String toString() { - StringBuilder out = new StringBuilder(); - for (int i = 0; i < byteValuedDiscreteFeatures.length; i++) { - if (out.length() > 0) - out.append(" "); - out.append((int) byteValuedDiscreteFeatures[i]); - } - for (int i = 0; i < shortValuedDiscreteFeatures.length; i++) { - if (out.length() > 0) - out.append(" "); - out.append((int) shortValuedDiscreteFeatures[i]); - } - for (int i = 0; i < continuousFeatures.length; i++) { - if (out.length() > 0) - out.append(" "); - out.append(continuousFeatures[i]); - } - return out.toString(); - } - -} diff --git a/marytts-unitselection/src/main/java/marytts/features/HalfPhoneTarget.java b/marytts-unitselection/src/main/java/marytts/features/HalfPhoneTarget.java deleted file mode 100644 index 8e29ea8d3f..0000000000 --- a/marytts-unitselection/src/main/java/marytts/features/HalfPhoneTarget.java +++ /dev/null @@ -1,60 +0,0 @@ -/** - * Copyright 2006 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.features; - -import org.w3c.dom.Element; - -public class HalfPhoneTarget extends Target { - protected boolean isLeftHalf; - - /** - * Create a target associated to the given segment item. - * - * @param name - * a name for the target, which may or may not coincide with the segment name. - * @param item - * the phone segment item in the Utterance structure, to be associated to this target - * @param isLeftHalf - * true if this target represents the left half of the phone, false if it represents the right half of the phone - */ - public HalfPhoneTarget(String name, Element maryxmlElement, boolean isLeftHalf) { - super(name, maryxmlElement); - this.isLeftHalf = isLeftHalf; - } - - /** - * Is this target the left half of a phone? - * - * @return - */ - public boolean isLeftHalf() { - return isLeftHalf; - } - - /** - * Is this target the right half of a phone? - * - * @return - */ - public boolean isRightHalf() { - return !isLeftHalf; - } - -} diff --git a/marytts-unitselection/src/main/java/marytts/features/MaryFeatureProcessor.java b/marytts-unitselection/src/main/java/marytts/features/MaryFeatureProcessor.java deleted file mode 100644 index 18a61c1338..0000000000 --- a/marytts-unitselection/src/main/java/marytts/features/MaryFeatureProcessor.java +++ /dev/null @@ -1,40 +0,0 @@ -/** - * Portions Copyright 2006 DFKI GmbH. - * Portions Copyright 2001 Sun Microsystems, Inc. - * Portions Copyright 1999-2001 Language Technologies Institute, - * Carnegie Mellon University. - * All Rights Reserved. Use is subject to license terms. - * - * Permission is hereby granted, free of charge, to use and distribute - * this software and its documentation without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of this work, and to - * permit persons to whom this work is furnished to do so, subject to - * the following conditions: - * - * 1. The code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * 2. Any modifications must be clearly marked as such. - * 3. Original authors' names are not deleted. - * 4. The authors' names are not used to endorse or promote products - * derived from this software without specific prior written - * permission. - * - * DFKI GMBH AND THE CONTRIBUTORS TO THIS WORK DISCLAIM ALL WARRANTIES WITH - * REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL DFKI GMBH NOR THE - * CONTRIBUTORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL - * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR - * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS - * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF - * THIS SOFTWARE. - */ -package marytts.features; - -/** - * Performs a specific type of processing on an item and returns an object. - */ -public interface MaryFeatureProcessor { - - public String getName(); -} diff --git a/marytts-unitselection/src/main/java/marytts/features/MaryGenericFeatureProcessors.java b/marytts-unitselection/src/main/java/marytts/features/MaryGenericFeatureProcessors.java deleted file mode 100644 index ba8dc4be0b..0000000000 --- a/marytts-unitselection/src/main/java/marytts/features/MaryGenericFeatureProcessors.java +++ /dev/null @@ -1,3055 +0,0 @@ -/** - * Portions Copyright 2006-2007 DFKI GmbH. - * Portions Copyright 2001 Sun Microsystems, Inc. - * Portions Copyright 1999-2001 Language Technologies Institute, - * Carnegie Mellon University. - * All Rights Reserved. Use is subject to license terms. - * - * Permission is hereby granted, free of charge, to use and distribute - * this software and its documentation without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of this work, and to - * permit persons to whom this work is furnished to do so, subject to - * the following conditions: - * - * 1. The code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * 2. Any modifications must be clearly marked as such. - * 3. Original authors' names are not deleted. - * 4. The authors' names are not used to endorse or promote products - * derived from this software without specific prior written - * permission. - * - * DFKI GMBH AND THE CONTRIBUTORS TO THIS WORK DISCLAIM ALL WARRANTIES WITH - * REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL DFKI GMBH NOR THE - * CONTRIBUTORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL - * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR - * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS - * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF - * THIS SOFTWARE. - */ - -package marytts.features; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashSet; -import java.util.Set; -import java.util.StringTokenizer; - -import marytts.datatypes.MaryXML; -import marytts.util.dom.MaryDomUtils; -import marytts.util.string.ByteStringTranslator; - -import org.w3c.dom.Document; -import org.w3c.dom.Element; -import org.w3c.dom.traversal.TreeWalker; - -/** - * A collection of feature processors that operate on Target objects. - * - * @author schroed - * - */ -public class MaryGenericFeatureProcessors { - /** - * Navigate from a target to an item. Classes implementing this interface will retrieve meaningful items given the target. - * - * @author Marc Schröder - */ - public static interface TargetElementNavigator { - /** - * Given the target, retrieve an XML Element. - * - * @param target - * @return an item selected according to this navigator, or null if there is no such item. - */ - public Element getElement(Target target); - } - - /** - * Retrieve the segment belonging to this target. - * - * @author Marc Schröder - * - */ - public static class SegmentNavigator implements TargetElementNavigator { - public Element getElement(Target target) { - return target.getMaryxmlElement(); - } - } - - /** - * Retrieve the segment preceding the segment which belongs to this target. - * - * @author Marc Schröder - * - */ - public static class PrevSegmentNavigator implements TargetElementNavigator { - public Element getElement(Target target) { - Element segment = target.getMaryxmlElement(); - if (segment == null) - return null; - Element sentence = (Element) MaryDomUtils.getAncestor(segment, MaryXML.SENTENCE); - if (sentence == null) - return null; - TreeWalker tw = MaryDomUtils.createTreeWalker(sentence, MaryXML.PHONE, MaryXML.BOUNDARY); - tw.setCurrentNode(segment); - Element previous = (Element) tw.previousNode(); - return previous; - } - } - - /** - * Retrieve the segment two before the segment which belongs to this target. - * - * @author Marc Schröder - * - */ - public static class PrevPrevSegmentNavigator implements TargetElementNavigator { - public Element getElement(Target target) { - Element segment = target.getMaryxmlElement(); - if (segment == null) - return null; - Element sentence = (Element) MaryDomUtils.getAncestor(segment, MaryXML.SENTENCE); - if (sentence == null) - return null; - TreeWalker tw = MaryDomUtils.createTreeWalker(sentence, MaryXML.PHONE, MaryXML.BOUNDARY); - tw.setCurrentNode(segment); - Element previous = (Element) tw.previousNode(); - Element pp = (Element) tw.previousNode(); - return pp; - } - } - - /** - * Retrieve the segment following the segment which belongs to this target. - * - * @author Marc Schröder - * - */ - public static class NextSegmentNavigator implements TargetElementNavigator { - public Element getElement(Target target) { - Element segment = target.getMaryxmlElement(); - if (segment == null) - return null; - Element sentence = (Element) MaryDomUtils.getAncestor(segment, MaryXML.SENTENCE); - if (sentence == null) - return null; - TreeWalker tw = MaryDomUtils.createTreeWalker(sentence, MaryXML.PHONE, MaryXML.BOUNDARY); - tw.setCurrentNode(segment); - Element next = (Element) tw.nextNode(); - return next; - } - } - - /** - * Retrieve the segment two after the segment which belongs to this target. - * - * @author Marc Schröder - * - */ - public static class NextNextSegmentNavigator implements TargetElementNavigator { - public Element getElement(Target target) { - Element segment = target.getMaryxmlElement(); - if (segment == null) - return null; - Element sentence = (Element) MaryDomUtils.getAncestor(segment, MaryXML.SENTENCE); - if (sentence == null) - return null; - TreeWalker tw = MaryDomUtils.createTreeWalker(sentence, MaryXML.PHONE, MaryXML.BOUNDARY); - tw.setCurrentNode(segment); - Element next = (Element) tw.nextNode(); - Element nn = (Element) tw.nextNode(); - return nn; - } - } - - /** - * Retrieve the first segment in the word to which this target belongs. - * - */ - public static class FirstSegmentInWordNavigator implements TargetElementNavigator { - public Element getElement(Target target) { - Element segment = target.getMaryxmlElement(); - if (segment == null) - return null; - Element word = (Element) MaryDomUtils.getAncestor(segment, MaryXML.TOKEN); - if (word == null) - return null; - TreeWalker tw = MaryDomUtils.createTreeWalker(word, MaryXML.PHONE); - Element first = (Element) tw.firstChild(); - if (first != null) { - assert first.getTagName().equals(MaryXML.PHONE) : "Unexpected tag name: expected " + MaryXML.PHONE + ", got " - + first.getTagName(); - } - return first; - } - } - - /** - * Retrieve the last segment in the word to which this target belongs. - * - */ - public static class LastSegmentInWordNavigator implements TargetElementNavigator { - public Element getElement(Target target) { - Element segment = target.getMaryxmlElement(); - if (segment == null) - return null; - Element word = (Element) MaryDomUtils.getAncestor(segment, MaryXML.TOKEN); - if (word == null) - return null; - TreeWalker tw = MaryDomUtils.createTreeWalker(word, MaryXML.PHONE); - Element last = (Element) tw.lastChild(); - if (last != null) { - assert last.getTagName().equals(MaryXML.PHONE) : "Unexpected tag name: expected " + MaryXML.PHONE + ", got " - + last.getTagName(); - } - return last; - } - } - - /** - * Retrieve the first syllable in the word to which this target belongs. - * - */ - public static class FirstSyllableInWordNavigator implements TargetElementNavigator { - public Element getElement(Target target) { - Element segment = target.getMaryxmlElement(); - if (segment == null) - return null; - Element word = (Element) MaryDomUtils.getAncestor(segment, MaryXML.TOKEN); - if (word == null) - return null; - TreeWalker tw = MaryDomUtils.createTreeWalker(word, MaryXML.SYLLABLE); - Element first = (Element) tw.firstChild(); - if (first != null) { - assert first.getTagName().equals(MaryXML.SYLLABLE) : "Unexpected tag name: expected " + MaryXML.SYLLABLE - + ", got " + first.getTagName(); - } - return first; - } - } - - /** - * Retrieve the last syllable in the word to which this target belongs. - * - */ - public static class LastSyllableInWordNavigator implements TargetElementNavigator { - public Element getElement(Target target) { - Element segment = target.getMaryxmlElement(); - if (segment == null) - return null; - Element word = (Element) MaryDomUtils.getAncestor(segment, MaryXML.TOKEN); - if (word == null) - return null; - TreeWalker tw = MaryDomUtils.createTreeWalker(word, MaryXML.SYLLABLE); - Element last = (Element) tw.lastChild(); - if (last != null) { - assert last.getTagName().equals(MaryXML.SYLLABLE) : "Unexpected tag name: expected " + MaryXML.SYLLABLE - + ", got " + last.getTagName(); - } - return last; - } - } - - /** - * Retrieve the syllable belonging to this target. - * - * @author Marc Schröder - * - */ - public static class SyllableNavigator implements TargetElementNavigator { - public Element getElement(Target target) { - Element segment = target.getMaryxmlElement(); - if (segment == null) - return null; - if (!segment.getTagName().equals(MaryXML.PHONE)) - return null; - Element syllable = (Element) segment.getParentNode(); - if (syllable != null) { - assert syllable.getTagName().equals(MaryXML.SYLLABLE) : "Unexpected tag name: expected " + MaryXML.SYLLABLE - + ", got " + syllable.getTagName(); - } - return syllable; - } - } - - /** - * Retrieve the syllable before the syllable belonging to this target. - * - * @author Marc Schröder - * - */ - public static class PrevSyllableNavigator implements TargetElementNavigator { - public Element getElement(Target target) { - Element segment = target.getMaryxmlElement(); - if (segment == null) - return null; - Element current; - if (segment.getTagName().equals(MaryXML.PHONE)) { - Element syllable = (Element) segment.getParentNode(); - if (syllable == null) - return null; - current = syllable; - } else { // boundary - current = segment; - } - Element sentence = (Element) MaryDomUtils.getAncestor(segment, MaryXML.SENTENCE); - if (sentence == null) - return null; - TreeWalker tw = MaryDomUtils.createTreeWalker(sentence, MaryXML.SYLLABLE); - tw.setCurrentNode(current); - Element previous = (Element) tw.previousNode(); - if (previous != null) { - assert previous.getTagName().equals(MaryXML.SYLLABLE) : "Unexpected tag name: expected " + MaryXML.SYLLABLE - + ", got " + previous.getTagName(); - } - return previous; - } - } - - /** - * Retrieve the syllable two before the syllable belonging to this target. - * - * @author Marc Schröder - * - */ - public static class PrevPrevSyllableNavigator implements TargetElementNavigator { - public Element getElement(Target target) { - Element segment = target.getMaryxmlElement(); - if (segment == null) - return null; - Element current; - if (segment.getTagName().equals(MaryXML.PHONE)) { - Element syllable = (Element) segment.getParentNode(); - if (syllable == null) - return null; - current = syllable; - } else { // boundary - current = segment; - } - Element sentence = (Element) MaryDomUtils.getAncestor(segment, MaryXML.SENTENCE); - if (sentence == null) - return null; - TreeWalker tw = MaryDomUtils.createTreeWalker(sentence, MaryXML.SYLLABLE); - tw.setCurrentNode(current); - Element previous = (Element) tw.previousNode(); - Element pp = (Element) tw.previousNode(); - if (pp != null) { - assert pp.getTagName().equals(MaryXML.SYLLABLE) : "Unexpected tag name: expected " + MaryXML.SYLLABLE + ", got " - + pp.getTagName(); - } - return pp; - } - } - - /** - * Retrieve the syllable following the syllable belonging to this target. - * - * @author Marc Schröder - * - */ - public static class NextSyllableNavigator implements TargetElementNavigator { - public Element getElement(Target target) { - Element segment = target.getMaryxmlElement(); - if (segment == null) - return null; - Element current; - if (segment.getTagName().equals(MaryXML.PHONE)) { - Element syllable = (Element) segment.getParentNode(); - if (syllable == null) - return null; - current = syllable; - } else { // boundary - current = segment; - } - Element sentence = (Element) MaryDomUtils.getAncestor(segment, MaryXML.SENTENCE); - if (sentence == null) - return null; - TreeWalker tw = MaryDomUtils.createTreeWalker(sentence, MaryXML.SYLLABLE); - tw.setCurrentNode(current); - Element next = (Element) tw.nextNode(); - if (next != null) { - assert next.getTagName().equals(MaryXML.SYLLABLE) : "Unexpected tag name: expected " + MaryXML.SYLLABLE - + ", got " + next.getTagName(); - } - return next; - } - } - - /** - * Retrieve the syllable two after the syllable belonging to this target. - * - * @author Marc Schröder - * - */ - public static class NextNextSyllableNavigator implements TargetElementNavigator { - public Element getElement(Target target) { - Element segment = target.getMaryxmlElement(); - if (segment == null) - return null; - Element current; - if (segment.getTagName().equals(MaryXML.PHONE)) { - Element syllable = (Element) segment.getParentNode(); - if (syllable == null) - return null; - current = syllable; - } else { // boundary - current = segment; - } - Element sentence = (Element) MaryDomUtils.getAncestor(segment, MaryXML.SENTENCE); - if (sentence == null) - return null; - TreeWalker tw = MaryDomUtils.createTreeWalker(sentence, MaryXML.SYLLABLE); - tw.setCurrentNode(current); - Element next = (Element) tw.nextNode(); - Element nn = (Element) tw.nextNode(); - if (nn != null) { - assert nn.getTagName().equals(MaryXML.SYLLABLE) : "Unexpected tag name: expected " + MaryXML.SYLLABLE + ", got " - + nn.getTagName(); - } - return nn; - } - } - - /** - * Retrieve the word belonging to this target. - * - * @author Marc Schröder - * - */ - public static class WordNavigator implements TargetElementNavigator { - public Element getElement(Target target) { - Element segment = target.getMaryxmlElement(); - if (segment == null) - return null; - Element word = (Element) MaryDomUtils.getAncestor(segment, MaryXML.TOKEN); - if (word != null) { - assert word.getTagName().equals(MaryXML.TOKEN) : "Unexpected tag name: expected " + MaryXML.TOKEN + ", got " - + word.getTagName(); - } - return word; - } - } - - /** Last syllable in phrase. */ - public static class LastSyllableInPhraseNavigator implements TargetElementNavigator { - public Element getElement(Target target) { - Element segment = target.getMaryxmlElement(); - if (segment == null) - return null; - Element phrase = (Element) MaryDomUtils.getAncestor(segment, MaryXML.PHRASE); - if (phrase == null) - return null; - TreeWalker tw = MaryDomUtils.createTreeWalker(phrase, MaryXML.SYLLABLE); - Element last = (Element) tw.lastChild(); - if (last != null) { - assert last.getTagName().equals(MaryXML.SYLLABLE) : "Unexpected tag name: expected " + MaryXML.SYLLABLE - + ", got " + last.getTagName(); - } - return last; - } - } - - public static class NextWordNavigator implements TargetElementNavigator { - public Element getElement(Target target) { - Element segment = target.getMaryxmlElement(); - if (segment == null) - return null; - Element current; - if (segment.getTagName().equals(MaryXML.PHONE)) { - Element word = (Element) MaryDomUtils.getAncestor(segment, MaryXML.TOKEN); - if (word == null) - return null; - current = word; - } else { // boundary - current = segment; - } - Element sentence = (Element) MaryDomUtils.getAncestor(segment, MaryXML.SENTENCE); - if (sentence == null) - return null; - TreeWalker tw = MaryDomUtils.createTreeWalker(sentence, MaryXML.TOKEN); - tw.setCurrentNode(current); - // The next word is the next token with a "ph" attribute: - Element nextWord = null; - Element nextToken; - while ((nextToken = (Element) tw.nextNode()) != null) { - if (nextToken.hasAttribute("ph")) { - nextWord = nextToken; - break; - } - } - if (nextWord != null) { - assert nextWord.getTagName().equals(MaryXML.TOKEN) : "Unexpected tag name: expected " + MaryXML.TOKEN + ", got " - + nextWord.getTagName(); - } - return nextWord; - } - } - - public static class PrevWordNavigator implements TargetElementNavigator { - public Element getElement(Target target) { - Element segment = target.getMaryxmlElement(); - if (segment == null) - return null; - Element current; - if (segment.getTagName().equals(MaryXML.PHONE)) { - Element word = (Element) MaryDomUtils.getAncestor(segment, MaryXML.TOKEN); - if (word == null) - return null; - current = word; - } else { // boundary - current = segment; - } - Element sentence = (Element) MaryDomUtils.getAncestor(segment, MaryXML.SENTENCE); - if (sentence == null) - return null; - TreeWalker tw = MaryDomUtils.createTreeWalker(sentence, MaryXML.TOKEN); - tw.setCurrentNode(current); - // The next word is the next token with a "ph" attribute: - Element prevWord = null; - Element prevToken; - while ((prevToken = (Element) tw.previousNode()) != null) { - if (prevToken.hasAttribute("ph")) { - prevWord = prevToken; - break; - } - } - if (prevWord != null) { - assert prevWord.getTagName().equals(MaryXML.TOKEN) : "Unexpected tag name: expected " + MaryXML.TOKEN + ", got " - + prevWord.getTagName(); - } - return prevWord; - } - } - - public static class FirstSegmentNextWordNavigator implements TargetElementNavigator { - public Element getElement(Target target) { - Element segment = target.getMaryxmlElement(); - if (segment == null) - return null; - Element current; - if (segment.getTagName().equals(MaryXML.PHONE)) { - Element word = (Element) MaryDomUtils.getAncestor(segment, MaryXML.TOKEN); - if (word == null) - return null; - current = word; - } else { // boundary - current = segment; - } - Element sentence = (Element) MaryDomUtils.getAncestor(segment, MaryXML.SENTENCE); - if (sentence == null) - return null; - TreeWalker tw = MaryDomUtils.createTreeWalker(sentence, MaryXML.TOKEN); - tw.setCurrentNode(current); - // The next word is the next token with a "ph" attribute: - Element nextWord = null; - Element nextToken; - while ((nextToken = (Element) tw.nextNode()) != null) { - if (nextToken.hasAttribute("ph")) { - nextWord = nextToken; - break; - } - } - if (nextWord == null) { - return null; - } - assert nextWord.getTagName().equals(MaryXML.TOKEN) : "Unexpected tag name: expected " + MaryXML.TOKEN + ", got " - + nextWord.getTagName(); - TreeWalker sw = MaryDomUtils.createTreeWalker(nextWord, MaryXML.PHONE); - Element first = (Element) sw.firstChild(); - if (first != null) { - assert first.getTagName().equals(MaryXML.PHONE) : "Unexpected tag name: expected " + MaryXML.PHONE + ", got " - + first.getTagName(); - } - return first; - } - } - - public static class LastWordInSentenceNavigator implements TargetElementNavigator { - public Element getElement(Target target) { - Element segment = target.getMaryxmlElement(); - if (segment == null) - return null; - Element sentence = (Element) MaryDomUtils.getAncestor(segment, MaryXML.SENTENCE); - if (sentence == null) - return null; - TreeWalker tw = MaryDomUtils.createTreeWalker(sentence, MaryXML.TOKEN); - Element lastWord = null; - Element lastToken = (Element) tw.lastChild(); - // The last word is the lastToken which has a "ph" attribute: - while (lastToken != null) { - if (lastToken.hasAttribute("ph")) { - lastWord = lastToken; - break; - } - lastToken = (Element) tw.previousNode(); - } - - if (lastWord != null) { - assert lastWord.getTagName().equals(MaryXML.TOKEN) : "Unexpected tag name: expected " + MaryXML.TOKEN + ", got " - + lastWord.getTagName(); - } - return lastWord; - } - } - - // no instances - protected MaryGenericFeatureProcessors() { - } - - /** - * flite never returns an int more than 19 from a feature processor, we duplicate that behavior in the processors so that our - * tests will match. let's keep this number as a constant for better overview - */ - private static final int RAIL_LIMIT = 19; - - private static final String[] ZERO_TO_NINETEEN = new String[] { "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", - "12", "13", "14", "15", "16", "17", "18", "19" }; - - /** - * Indicate whether a unit is an edge unit, which is never the case for a target. - */ - public static class Edge implements ByteValuedFeatureProcessor { - public String getName() { - return "edge"; - } - - public String[] getValues() { - return new String[] { "0", "start", "end" }; - } - - /** - * This processor always returns 0 for targets. - */ - public byte process(Target target) { - return (byte) 0; - } - - } - - /** - * Is the given half phone target a left or a right half? - * - * @author Marc Schröder - * - */ - public static class HalfPhoneLeftRight implements ByteValuedFeatureProcessor { - protected ByteStringTranslator values; - - /** - * Initialise a HalfPhoneLeftRight feature processor. - */ - public HalfPhoneLeftRight() { - this.values = new ByteStringTranslator(new String[] { "0", "L", "R" }); - } - - public String getName() { - return "halfphone_lr"; - } - - public String[] getValues() { - return values.getStringValues(); - } - - public byte process(Target target) { - if (!(target instanceof HalfPhoneTarget)) - return 0; - HalfPhoneTarget hpTarget = (HalfPhoneTarget) target; - String value = (hpTarget.isLeftHalf() ? "L" : "R"); - return values.get(value); - } - } - - /** - * Sentence Style for the given target - * - * @author Sathish Chandra Pammi - * - */ - - public static class Style implements ByteValuedFeatureProcessor { - protected ByteStringTranslator values; - protected TargetElementNavigator navigator; - protected final String styleTagName = "style"; - - /** - * Initialize a speaking Style feature processor. - */ - public Style() { - this.values = new ByteStringTranslator(new String[] { "0", "neutral", "poker", "happy", "sad", "angry", "excited" }); - this.navigator = new SegmentNavigator(); - } - - public String getName() { - return styleTagName; - } - - public String[] getValues() { - return values.getStringValues(); - } - - public byte process(Target target) { - String style = null; - Element segment = target.getMaryxmlElement(); - if (segment != null) { - Element prosody = (Element) MaryDomUtils.getClosestAncestorWithAttribute(segment, MaryXML.PROSODY, styleTagName); - if (prosody != null) { - style = prosody.getAttribute(styleTagName); - } - } - if (style == null || style.equals("")) - style = "0"; - if (values.contains(style)) { - return values.get(style); - } else { // silently ignore unknown values - return 0; - } - } - } - - /** - * Checks to see if the given syllable is accented. - */ - public static class Accented implements ByteValuedFeatureProcessor { - protected String name; - protected TargetElementNavigator navigator; - - public Accented(String name, TargetElementNavigator syllableNavigator) { - this.name = name; - this.navigator = syllableNavigator; - } - - public String getName() { - return name; - } - - public String[] getValues() { - return new String[] { "0", "1" }; - } - - /** - * Performs some processing on the given item. - * - * @param target - * the target to process - * @return "1" if the syllable is accented; otherwise "0" - */ - public byte process(Target target) { - Element syllable = navigator.getElement(target); - if (syllable != null && syllable.hasAttribute("accent")) { - return (byte) 1; - } else { - return (byte) 0; - } - } - } - - /** - * Checks to see if the given syllable is stressed. - */ - public static class Stressed implements ByteValuedFeatureProcessor { - protected String name; - protected TargetElementNavigator navigator; - - public Stressed(String name, TargetElementNavigator syllableNavigator) { - this.name = name; - this.navigator = syllableNavigator; - } - - public String getName() { - return name; - } - - public String[] getValues() { - return new String[] { "0", "1" }; - } - - /** - * Performs some processing on the given item. - * - * @param target - * the target to process - * @return "1" if the syllable is stressed; otherwise "0" - */ - public byte process(Target target) { - Element syllable = navigator.getElement(target); - if (syllable == null) - return 0; - String value = syllable.getAttribute("stress"); - if (value.equals("")) - return 0; - byte stressValue = Byte.parseByte(value); - if (stressValue > 1) { - // out of range, set to 1 - stressValue = 1; - } - return stressValue; - } - } - - /** - * Syllable tone for the given target - * - * @author sathish pammi - * - */ - public static class SyllableTone implements ByteValuedFeatureProcessor { - protected String name; - protected TargetElementNavigator navigator; - - public SyllableTone(String name, TargetElementNavigator syllableNavigator) { - this.name = name; - this.navigator = syllableNavigator; - } - - public String getName() { - return name; - } - - public String[] getValues() { - return new String[] { "0", "1", "2", "3", "4" }; - } - - /** - * Performs some processing on the given item. - * - * @param target - * the target to process - * @return tone value - */ - public byte process(Target target) { - Element syllable = navigator.getElement(target); - if (syllable == null) - return 0; - String value = syllable.getAttribute("tone"); - if (value.equals("")) - return 0; - byte toneValue = Byte.parseByte(value); - if (toneValue > 4 || toneValue < 1) { - // out of range, set to 0 - toneValue = 0; - } - return toneValue; - } - } - - /** - * Returns as a byte the number of phrases in the current sentence. - */ - public static class SentenceNumPhrases implements ByteValuedFeatureProcessor { - public SentenceNumPhrases() { - } - - public String getName() { - return "sentence_numphrases"; - } - - public String[] getValues() { - return ZERO_TO_NINETEEN; - } - - /** - * - * @return the number of phrases in the sentence - */ - public byte process(Target target) { - Element segment = target.getMaryxmlElement(); - if (segment == null) - return (byte) 0; - Element sentence = (Element) MaryDomUtils.getAncestor(segment, MaryXML.SENTENCE); - if (sentence == null) - return (byte) 0; - TreeWalker tw = MaryDomUtils.createTreeWalker(sentence, MaryXML.PHRASE); - int count = 0; - Element e; - while ((e = (Element) tw.nextNode()) != null && count < RAIL_LIMIT) { - count++; - } - return (byte) count; - } - } - - /** - * Returns as an Integer the number of words in the current sentence. This is a feature processor. A feature processor takes - * an item, performs some sort of processing on the item and returns an object. - */ - public static class SentenceNumWords implements ByteValuedFeatureProcessor { - public SentenceNumWords() { - } - - public String getName() { - return "sentence_numwords"; - } - - public String[] getValues() { - return ZERO_TO_NINETEEN; - } - - /** - * @return the number of words in the sentence - */ - public byte process(Target target) { - Element segment = target.getMaryxmlElement(); - if (segment == null) - return (byte) 0; - Element sentence = (Element) MaryDomUtils.getAncestor(segment, MaryXML.SENTENCE); - if (sentence == null) - return (byte) 0; - TreeWalker tw = MaryDomUtils.createTreeWalker(sentence, MaryXML.TOKEN); - int count = 0; - Element e; - while ((e = (Element) tw.nextNode()) != null && count < RAIL_LIMIT) { - // only tokens with a "ph" attribute count as words: - if (e.hasAttribute("ph")) - count++; - } - return (byte) count; - } - } - - /** - * Returns as a byte the number of phrases in the current sentence. - */ - public static class PhraseNumSyls implements ByteValuedFeatureProcessor { - public PhraseNumSyls() { - } - - public String getName() { - return "phrase_numsyls"; - } - - public String[] getValues() { - return ZERO_TO_NINETEEN; - } - - /** - * @return the number of words in the phrase - */ - public byte process(Target target) { - Element segment = target.getMaryxmlElement(); - if (segment == null) - return (byte) 0; - Element phrase = (Element) MaryDomUtils.getAncestor(segment, MaryXML.PHRASE); - if (phrase == null) - return (byte) 0; - TreeWalker tw = MaryDomUtils.createTreeWalker(phrase, MaryXML.SYLLABLE); - int count = 0; - Element e; - while ((e = (Element) tw.nextNode()) != null && count < RAIL_LIMIT) { - count++; - } - return (byte) count; - } - } - - /** - * Returns as a byte the number of words in the current phrase. - */ - public static class PhraseNumWords implements ByteValuedFeatureProcessor { - public PhraseNumWords() { - } - - public String getName() { - return "phrase_numwords"; - } - - public String[] getValues() { - return ZERO_TO_NINETEEN; - } - - /** - * Performs some processing on the given item. - * - * @param item - * the item to process - * - * @return the number of words in the phrase - */ - public byte process(Target target) { - Element segment = target.getMaryxmlElement(); - if (segment == null) - return (byte) 0; - Element phrase = (Element) MaryDomUtils.getAncestor(segment, MaryXML.PHRASE); - if (phrase == null) - return (byte) 0; - TreeWalker tw = MaryDomUtils.createTreeWalker(phrase, MaryXML.TOKEN); - int count = 0; - Element e; - while ((e = (Element) tw.nextNode()) != null && count < RAIL_LIMIT) { - count++; - } - return (byte) count; - } - } - - /** - * Returns as an Integer the number of syllables in the given word. This is a feature processor. A feature processor takes an - * item, performs some sort of processing on the item and returns an object. - */ - public static class WordNumSyls implements ByteValuedFeatureProcessor { - public WordNumSyls() { - } - - public String getName() { - return "word_numsyls"; - } - - public String[] getValues() { - return ZERO_TO_NINETEEN; - } - - /** - * @return the number of syllables in the given word - */ - public byte process(Target target) { - Element segment = target.getMaryxmlElement(); - if (segment == null) - return (byte) 0; - Element word = (Element) MaryDomUtils.getAncestor(segment, MaryXML.TOKEN); - if (word == null) - return (byte) 0; - TreeWalker tw = MaryDomUtils.createTreeWalker(word, MaryXML.SYLLABLE); - int count = 0; - Element e; - while ((e = (Element) tw.nextNode()) != null && count < RAIL_LIMIT) { - count++; - } - return (byte) count; - } - } - - /** - * Returns as a byte the number of segments in the given word. - */ - public static class WordNumSegs implements ByteValuedFeatureProcessor { - public WordNumSegs() { - } - - public String getName() { - return "word_numsegs"; - } - - public String[] getValues() { - return ZERO_TO_NINETEEN; - } - - /** - * @return the number of segments in the given word - */ - public byte process(Target target) { - Element segment = target.getMaryxmlElement(); - if (segment == null) - return (byte) 0; - Element word = (Element) MaryDomUtils.getAncestor(segment, MaryXML.TOKEN); - if (word == null) - return (byte) 0; - TreeWalker tw = MaryDomUtils.createTreeWalker(word, MaryXML.PHONE); - int count = 0; - Element e; - while ((e = (Element) tw.nextNode()) != null && count < RAIL_LIMIT) { - count++; - } - return (byte) count; - } - } - - /** - * Returns as an Integer the number of segments in the current syllable. This is a feature processor. A feature processor - * takes an item, performs some sort of processing on the item and returns an object. - */ - public static class SylNumSegs implements ByteValuedFeatureProcessor { - public SylNumSegs() { - } - - public String getName() { - return "syl_numsegs"; - } - - public String[] getValues() { - return ZERO_TO_NINETEEN; - } - - /** - * Performs some processing on the given item. - * - * @param item - * the item to process - * - * @return the number of segments in the given word - */ - public byte process(Target target) { - Element segment = target.getMaryxmlElement(); - if (segment == null) - return (byte) 0; - if (!segment.getTagName().equals(MaryXML.PHONE)) - return 0; - Element syllable = (Element) segment.getParentNode(); - if (syllable == null) - return (byte) 0; - TreeWalker tw = MaryDomUtils.createTreeWalker(syllable, MaryXML.PHONE); - int count = 0; - Element e; - while ((e = (Element) tw.nextNode()) != null && count < RAIL_LIMIT) { - count++; - } - return (byte) count; - } - } - - /** - * @deprecated, use SegsFromSylStart instead - */ - public static class PosInSyl extends SegsFromSylStart { - public PosInSyl() { - super(); - } - - public String getName() { - return "pos_in_syl"; - } - } - - /** - * Finds the position of the phone in the syllable. - */ - public static class SegsFromSylStart implements ByteValuedFeatureProcessor { - public SegsFromSylStart() { - } - - public String getName() { - return "segs_from_syl_start"; - } - - public String[] getValues() { - return ZERO_TO_NINETEEN; - } - - /** - * @param target - * the target to process - * @return the position of the phone in the syllable - */ - public byte process(Target target) { - Element segment = target.getMaryxmlElement(); - if (segment == null) - return (byte) 0; - if (!segment.getTagName().equals(MaryXML.PHONE)) - return 0; - int count = 0; - Element e = segment; - while ((e = MaryDomUtils.getPreviousSiblingElement(e)) != null && count < RAIL_LIMIT) { - count++; - } - return (byte) count; - } - } - - /** - * Finds the position of the phone from the end of the syllable. - */ - public static class SegsFromSylEnd implements ByteValuedFeatureProcessor { - public SegsFromSylEnd() { - } - - public String getName() { - return "segs_from_syl_end"; - } - - public String[] getValues() { - return ZERO_TO_NINETEEN; - } - - /** - * @param target - * the target to process - * @return the position of the phone in the syllable - */ - public byte process(Target target) { - Element segment = target.getMaryxmlElement(); - if (segment == null) - return (byte) 0; - if (!segment.getTagName().equals(MaryXML.PHONE)) - return 0; - int count = 0; - Element e = segment; - while ((e = MaryDomUtils.getNextSiblingElement(e)) != null && count < RAIL_LIMIT) { - count++; - } - return (byte) count; - } - } - - /** - * Finds the position of the segment from the start of the word. - */ - public static class SegsFromWordStart implements ByteValuedFeatureProcessor { - public SegsFromWordStart() { - } - - public String getName() { - return "segs_from_word_start"; - } - - public String[] getValues() { - return ZERO_TO_NINETEEN; - } - - /** - * @param target - * the target to process - * @return the position of the phone in the syllable - */ - public byte process(Target target) { - Element segment = target.getMaryxmlElement(); - if (segment == null) - return (byte) 0; - Element word = (Element) MaryDomUtils.getAncestor(segment, MaryXML.TOKEN); - if (word == null) - return (byte) 0; - TreeWalker tw = MaryDomUtils.createTreeWalker(word, MaryXML.PHONE); - tw.setCurrentNode(segment); - int count = 0; - Element e; - while ((e = (Element) tw.previousNode()) != null && count < RAIL_LIMIT) { - count++; - } - return (byte) count; - } - } - - /** - * Finds the position of the segment from the end of the word. - */ - public static class SegsFromWordEnd implements ByteValuedFeatureProcessor { - public SegsFromWordEnd() { - } - - public String getName() { - return "segs_from_word_end"; - } - - public String[] getValues() { - return ZERO_TO_NINETEEN; - } - - /** - * Performs some processing on the given item. - * - * @param target - * the target to process - * @return the position of the phone in the syllable - */ - public byte process(Target target) { - Element segment = target.getMaryxmlElement(); - if (segment == null) - return (byte) 0; - Element word = (Element) MaryDomUtils.getAncestor(segment, MaryXML.TOKEN); - if (word == null) - return (byte) 0; - TreeWalker tw = MaryDomUtils.createTreeWalker(word, MaryXML.PHONE); - tw.setCurrentNode(segment); - int count = 0; - Element e; - while ((e = (Element) tw.nextNode()) != null && count < RAIL_LIMIT) { - count++; - } - return (byte) count; - } - } - - /** - * Finds the position of the syllable from the start of the word. - */ - public static class SylsFromWordStart implements ByteValuedFeatureProcessor { - public SylsFromWordStart() { - } - - public String getName() { - return "syls_from_word_start"; - } - - public String[] getValues() { - return ZERO_TO_NINETEEN; - } - - /** - * @param target - * the target to process - * @return the position of the syllable in the word - */ - public byte process(Target target) { - Element segment = target.getMaryxmlElement(); - if (segment == null) - return (byte) 0; - if (!segment.getTagName().equals(MaryXML.PHONE)) - return 0; - Element syllable = (Element) segment.getParentNode(); - if (syllable == null) - return (byte) 0; - int count = 0; - Element e = syllable; - while ((e = MaryDomUtils.getPreviousSiblingElement(e)) != null && count < RAIL_LIMIT) { - count++; - } - return (byte) count; - } - } - - /** - * Finds the position of the syllable from the end of the word. - */ - public static class SylsFromWordEnd implements ByteValuedFeatureProcessor { - public SylsFromWordEnd() { - } - - public String getName() { - return "syls_from_word_end"; - } - - public String[] getValues() { - return ZERO_TO_NINETEEN; - } - - /** - * @param target - * the target to process - * @return the position of the phone in the syllable - */ - public byte process(Target target) { - Element segment = target.getMaryxmlElement(); - if (segment == null) - return (byte) 0; - if (!segment.getTagName().equals(MaryXML.PHONE)) - return 0; - Element syllable = (Element) segment.getParentNode(); - if (syllable == null) - return (byte) 0; - int count = 0; - Element e = syllable; - while ((e = MaryDomUtils.getNextSiblingElement(e)) != null && count < RAIL_LIMIT) { - count++; - } - return (byte) count; - } - } - - /** - * Determines the break level after this syllable. - */ - public static class SylBreak implements ByteValuedFeatureProcessor { - protected String name; - protected TargetElementNavigator navigator; - - public SylBreak(String name, TargetElementNavigator syllableNavigator) { - this.name = name; - this.navigator = syllableNavigator; - } - - public String getName() { - return name; - } - - /** - * "4" for a big break, "3" for a break; "1" = word-final; "0" = within-word - */ - public String[] getValues() { - return new String[] { "0", "1", "unused", "3", "4" }; - } - - /** - * @param target - * the target to process - * @return the break level after the syllable returned by syllableNavigator - */ - public byte process(Target target) { - Element syllable = navigator.getElement(target); - if (syllable == null) - return 0; - // is there another syllable following in the token? - if (MaryDomUtils.getNextSiblingElement(syllable) != null) - return 0; - // else, it is at least word-final. - Element word = (Element) syllable.getParentNode(); - if (word == null) - return 0; - assert word.getTagName().equals(MaryXML.TOKEN) : "Unexpected tag name: expected " + MaryXML.TOKEN + ", got " - + word.getTagName(); - Element sentence = (Element) MaryDomUtils.getAncestor(word, MaryXML.SENTENCE); - if (sentence == null) - return 0; - TreeWalker tw = MaryDomUtils.createTreeWalker(sentence, MaryXML.TOKEN, MaryXML.BOUNDARY); - tw.setCurrentNode(word); - // The next word is the next token with a "ph" attribute: - Element e; - while ((e = (Element) tw.nextNode()) != null) { - if (e.getTagName().equals(MaryXML.BOUNDARY) || e.getTagName().equals(MaryXML.TOKEN) && e.hasAttribute("ph")) - break; - } - if (e == null) { - // we are the last token in the sentence, but there is no boundary... - // OK, let's say it is sentence-final anyway: - return 4; - } - if (e.getTagName().equals(MaryXML.TOKEN)) { - // a word follows - return 1; - } - assert e.getTagName().equals(MaryXML.BOUNDARY) : "Unexpected tag name: expected " + MaryXML.BOUNDARY + ", got " - + e.getTagName(); - String bi = e.getAttribute("breakindex"); - if (bi.equals("")) { - // no breakindex - return 1; - } - try { - int ibi = Integer.parseInt(bi); - if (ibi >= 4) - return 4; - if (ibi == 3) - return 3; - } catch (NumberFormatException nfe) { - } - // default: a word boundary - return 1; - } - } - - /** - * Classifies the the syllable as single, initial, mid or final. - */ - public static class PositionType implements ByteValuedFeatureProcessor { - protected TargetElementNavigator navigator; - protected ByteStringTranslator values; - - public PositionType() { - values = new ByteStringTranslator(new String[] { "0", "single", "final", "initial", "mid" }); - navigator = new SyllableNavigator(); - } - - public String getName() { - return "position_type"; - } - - public String[] getValues() { - return values.getStringValues(); - } - - /** - * @param target - * the target to process - * @return classifies the syllable as "single", "final", "initial" or "mid" - */ - public byte process(Target target) { - Element syllable = navigator.getElement(target); - if (syllable == null) - return 0; - String type; - if (MaryDomUtils.getNextSiblingElement(syllable) == null) { - if (MaryDomUtils.getPreviousSiblingElement(syllable) == null) { - type = "single"; - } else { - type = "final"; - } - } else if (MaryDomUtils.getPreviousSiblingElement(syllable) == null) { - type = "initial"; - } else { - type = "mid"; - } - return values.get(type); - } - } - - /** - * Checks if segment is a pause. - */ - public static class IsPause implements ByteValuedFeatureProcessor { - protected TargetElementNavigator navigator; - protected String name; - - public IsPause(String name, TargetElementNavigator segmentNavigator) { - this.name = name; - this.navigator = segmentNavigator; - } - - public String getName() { - return name; - } - - public String[] getValues() { - return new String[] { "0", "1" }; - } - - /** - * Check if segment is a pause - * - * @param target - * the target to process - * @return 0 if false, 1 if true - */ - public byte process(Target target) { - Element seg = navigator.getElement(target); - if (seg == null) - return 0; - if (seg.getTagName().equals(MaryXML.BOUNDARY)) - return 1; - return 0; - } - } - - public static class BreakIndex implements ByteValuedFeatureProcessor { - protected ByteStringTranslator values; - - public BreakIndex() { - values = new ByteStringTranslator(new String[] { "0", "1", "2", "3", "4", "5", "6" }); - } - - public String getName() { - return "breakindex"; - } - - public String[] getValues() { - return values.getStringValues(); - } - - public byte process(Target target) { - Element segment = target.getMaryxmlElement(); - if (segment == null) - return 0; - Element word = (Element) MaryDomUtils.getAncestor(segment, MaryXML.TOKEN); - if (word == null) - return 0; - // is there another segment following in the token? - TreeWalker tww = MaryDomUtils.createTreeWalker(word, MaryXML.PHONE); - tww.setCurrentNode(segment); - if (tww.nextNode() != null) - return 0; - // else, it is at least word-final. - Element sentence = (Element) MaryDomUtils.getAncestor(word, MaryXML.SENTENCE); - if (sentence == null) - return 0; - TreeWalker tw = MaryDomUtils.createTreeWalker(sentence, MaryXML.TOKEN, MaryXML.BOUNDARY); - tw.setCurrentNode(word); - // The next word is the next token with a "ph" attribute: - Element e; - while ((e = (Element) tw.nextNode()) != null) { - if (e.getTagName().equals(MaryXML.BOUNDARY) || e.getTagName().equals(MaryXML.TOKEN) && e.hasAttribute("ph")) - break; - } - if (e == null) { - // we are the last token in the sentence, but there is no boundary... - // OK, let's say it is sentence-final anyway: - return 4; - } - if (e.getTagName().equals(MaryXML.TOKEN)) { - // a word follows - return 1; - } - assert e.getTagName().equals(MaryXML.BOUNDARY) : "Unexpected tag name: expected " + MaryXML.BOUNDARY + ", got " - + e.getTagName(); - String bi = e.getAttribute("breakindex"); - if (bi.equals("")) { - // no breakindex - return 1; - } - try { - int ibi = Integer.parseInt(bi); - if (ibi > 6) - ibi = 6; - if (ibi < 2) - ibi = 2; - return (byte) ibi; - } catch (NumberFormatException nfe) { - } - // default: a word boundary - return 1; - } - } - - /** - * The ToBI accent of the current syllable. - */ - public static class TobiAccent implements ByteValuedFeatureProcessor { - protected String name; - protected TargetElementNavigator navigator; - protected ByteStringTranslator values; - - public TobiAccent(String name, TargetElementNavigator syllableNavigator) { - this.name = name; - this.navigator = syllableNavigator; - this.values = new ByteStringTranslator(new String[] { "0", "*", "H*", "!H*", "^H*", "L*", "L+H*", "L*+H", "L+!H*", - "L*+!H", "L+^H*", "L*+^H", "H+L*", "H+!H*", "H+^H*", "!H+!H*", "^H+!H*", "^H+^H*", "H*+L", "!H*+L" }); - } - - public String getName() { - return name; - } - - public String[] getValues() { - return values.getStringValues(); - } - - /** - * For the given syllable item, return its tobi accent, or 0 if there is none. - */ - public byte process(Target target) { - Element syllable = navigator.getElement(target); - if (syllable == null) - return 0; - String accent = syllable.getAttribute("accent"); - if (accent.equals("")) { - return 0; - } - return values.get(accent); - } - } - - /** - * The ToBI endtone associated with the current syllable. - */ - public static class TobiEndtone implements ByteValuedFeatureProcessor { - protected String name; - protected TargetElementNavigator navigator; - protected ByteStringTranslator values; - - public TobiEndtone(String name, TargetElementNavigator syllableNavigator) { - this.name = name; - this.navigator = syllableNavigator; - this.values = new ByteStringTranslator(new String[] { "0", "H-", "!H-", "L-", "H-%", "!H-%", "H-^H%", "!H-^H%", - "L-H%", "L-%", "L-L%", "H-H%", "H-L%" }); - } - - public String getName() { - return name; - } - - public String[] getValues() { - return values.getStringValues(); - } - - /** - * For the given syllable item, return its tobi end tone, or 0 if there is none. - */ - public byte process(Target target) { - Element syllable = navigator.getElement(target); - if (syllable == null) - return 0; - Element sentence = (Element) MaryDomUtils.getAncestor(syllable, MaryXML.SENTENCE); - if (sentence == null) - return 0; - TreeWalker tw = MaryDomUtils.createTreeWalker(sentence, MaryXML.SYLLABLE, MaryXML.BOUNDARY); - tw.setCurrentNode(syllable); - Element e = (Element) tw.nextNode(); - if (e == null) - return 0; - if (e.getTagName().equals(MaryXML.SYLLABLE)) - return 0; - assert e.getTagName().equals(MaryXML.BOUNDARY) : "Unexpected tag name: expected " + MaryXML.BOUNDARY + ", got " - + e.getTagName(); - String endtone = e.getAttribute("tone"); - if (endtone.equals("")) { - return 0; - } - return values.get(endtone); - } - } - - /** - * The next ToBI accent following the current syllable in the current phrase. - */ - public static class NextAccent extends TobiAccent { - public NextAccent() { - super("next_accent", null); - } - - /** - * Search for an accented syllable, and return its tobi accent, or 0 if there is none. - */ - public byte process(Target target) { - Element segment = target.getMaryxmlElement(); - if (segment == null) - return 0; - Element current; - if (segment.getTagName().equals(MaryXML.PHONE)) { - Element syllable = (Element) segment.getParentNode(); - if (syllable == null) - return 0; - current = syllable; - } else { // boundary - current = segment; - } - Element phrase = (Element) MaryDomUtils.getAncestor(current, MaryXML.PHRASE); - if (phrase == null) - return 0; - TreeWalker tw = MaryDomUtils.createTreeWalker(phrase, MaryXML.SYLLABLE); - tw.setCurrentNode(current); - Element s; - while ((s = (Element) tw.nextNode()) != null) { - if (s.hasAttribute("accent")) { - String accent = s.getAttribute("accent"); - return values.get(accent); - } - } - return 0; - } - } - - /** - * The previous ToBI accent preceding the current syllable in the current phrase. - */ - public static class PrevAccent extends TobiAccent { - public PrevAccent() { - super("prev_accent", null); - } - - /** - * Search for an accented syllable, and return its tobi accent, or 0 if there is none. - */ - public byte process(Target target) { - Element segment = target.getMaryxmlElement(); - if (segment == null) - return 0; - Element current; - if (segment.getTagName().equals(MaryXML.PHONE)) { - Element syllable = (Element) segment.getParentNode(); - if (syllable == null) - return 0; - current = syllable; - } else { // boundary - current = segment; - } - Element phrase = (Element) MaryDomUtils.getAncestor(current, MaryXML.PHRASE); - if (phrase == null) - return 0; - TreeWalker tw = MaryDomUtils.createTreeWalker(phrase, MaryXML.SYLLABLE); - tw.setCurrentNode(current); - Element s; - while ((s = (Element) tw.previousNode()) != null) { - if (s.hasAttribute("accent")) { - String accent = s.getAttribute("accent"); - return values.get(accent); - } - } - return 0; - } - } - - /** - * The ToBI endtone associated with the last syllable of the current phrase. - */ - public static class PhraseEndtone extends TobiEndtone { - public PhraseEndtone() { - super("phrase_endtone", new LastSyllableInPhraseNavigator()); - } - } - - /** - * The ToBI endtone associated with the last syllable of the previous phrase. - */ - public static class PrevPhraseEndtone extends TobiEndtone { - public PrevPhraseEndtone() { - super("prev_phrase_endtone", null); - } - - public byte process(Target target) { - Element segment = target.getMaryxmlElement(); - if (segment == null) - return 0; - Element phrase = (Element) MaryDomUtils.getAncestor(segment, MaryXML.PHRASE); - if (phrase == null) - return 0; - Document doc = phrase.getOwnerDocument(); - TreeWalker tw = MaryDomUtils.createTreeWalker(doc, doc, MaryXML.BOUNDARY); - tw.setCurrentNode(phrase); - Element boundary = (Element) tw.previousNode(); - if (boundary == null) - return 0; - String endtone = boundary.getAttribute("tone"); - if (endtone.equals("")) { - return 0; - } - return values.get(endtone); - } - } - - /** - * Counts the number of syllables since the start of the phrase. - */ - public static class SylsFromPhraseStart implements ByteValuedFeatureProcessor { - public SylsFromPhraseStart() { - } - - public String getName() { - return "syls_from_phrase_start"; - } - - public String[] getValues() { - return ZERO_TO_NINETEEN; - } - - /** - * @param target - * the target to process - * @return the number of syllables since the last major break - */ - public byte process(Target target) { - Element segment = target.getMaryxmlElement(); - if (segment == null) - return 0; - Element phrase = (Element) MaryDomUtils.getAncestor(segment, MaryXML.PHRASE); - if (phrase == null) - return 0; - int count = 0; - TreeWalker tw = MaryDomUtils.createTreeWalker(phrase, MaryXML.SYLLABLE); - Element syllable = (Element) MaryDomUtils.getAncestor(segment, MaryXML.SYLLABLE); - if (syllable != null) { - tw.setCurrentNode(syllable); - } else { - tw.setCurrentNode(segment); - } - Element e; - while ((e = (Element) tw.previousNode()) != null && count < RAIL_LIMIT) { - count++; - } - return (byte) count; - } - } - - /** - * Counts the number of syllables until the end of the phrase. - */ - public static class SylsFromPhraseEnd implements ByteValuedFeatureProcessor { - public SylsFromPhraseEnd() { - } - - public String getName() { - return "syls_from_phrase_end"; - } - - public String[] getValues() { - return ZERO_TO_NINETEEN; - } - - /** - * @param target - * the target to process - * @return the number of accented syllables since the last major break - */ - public byte process(Target target) { - Element segment = target.getMaryxmlElement(); - if (segment == null) - return 0; - Element phrase = (Element) MaryDomUtils.getAncestor(segment, MaryXML.PHRASE); - if (phrase == null) - return 0; - int count = 0; - TreeWalker tw = MaryDomUtils.createTreeWalker(phrase, MaryXML.SYLLABLE); - tw.setCurrentNode(segment); - Element e; - while ((e = (Element) tw.nextNode()) != null && count < RAIL_LIMIT) { - count++; - } - return (byte) count; - } - } - - /** - * Counts the number of stressed syllables since the start of the phrase. - */ - public static class StressedSylsFromPhraseStart implements ByteValuedFeatureProcessor { - public StressedSylsFromPhraseStart() { - } - - public String getName() { - return "stressed_syls_from_phrase_start"; - } - - public String[] getValues() { - return ZERO_TO_NINETEEN; - } - - /** - * @param target - * the target to process - * @return the number of stressed syllables since the last major break - */ - public byte process(Target target) { - Element segment = target.getMaryxmlElement(); - if (segment == null) - return 0; - Element phrase = (Element) MaryDomUtils.getAncestor(segment, MaryXML.PHRASE); - if (phrase == null) - return 0; - int count = 0; - TreeWalker tw = MaryDomUtils.createTreeWalker(phrase, MaryXML.SYLLABLE); - Element syllable = (Element) MaryDomUtils.getAncestor(segment, MaryXML.SYLLABLE); - if (syllable != null) { - tw.setCurrentNode(syllable); - } else { - tw.setCurrentNode(segment); - } - Element e; - while ((e = (Element) tw.previousNode()) != null && count < RAIL_LIMIT) { - String stress = e.getAttribute("stress"); - if (stress.equals("1")) - count++; - } - return (byte) count; - } - } - - /** - * Counts the number of stressed syllables until the end of the phrase. - */ - public static class StressedSylsFromPhraseEnd implements ByteValuedFeatureProcessor { - public StressedSylsFromPhraseEnd() { - } - - public String getName() { - return "stressed_syls_from_phrase_end"; - } - - public String[] getValues() { - return ZERO_TO_NINETEEN; - } - - /** - * @param target - * the target to process - * @return the number of stressed syllables since the last major break - */ - public byte process(Target target) { - Element segment = target.getMaryxmlElement(); - if (segment == null) - return 0; - Element phrase = (Element) MaryDomUtils.getAncestor(segment, MaryXML.PHRASE); - if (phrase == null) - return 0; - int count = 0; - TreeWalker tw = MaryDomUtils.createTreeWalker(phrase, MaryXML.SYLLABLE); - tw.setCurrentNode(segment); - Element e; - while ((e = (Element) tw.nextNode()) != null && count < RAIL_LIMIT) { - String stress = e.getAttribute("stress"); - if (stress.equals("1")) - count++; - } - return (byte) count; - } - } - - /** - * Counts the number of accented syllables since the start of the phrase. - */ - public static class AccentedSylsFromPhraseStart implements ByteValuedFeatureProcessor { - public AccentedSylsFromPhraseStart() { - } - - public String getName() { - return "accented_syls_from_phrase_start"; - } - - public String[] getValues() { - return ZERO_TO_NINETEEN; - } - - /** - * @param target - * the target to process - * @return the number of accented syllables since the last major break - */ - public byte process(Target target) { - Element segment = target.getMaryxmlElement(); - if (segment == null) - return 0; - Element phrase = (Element) MaryDomUtils.getAncestor(segment, MaryXML.PHRASE); - if (phrase == null) - return 0; - int count = 0; - TreeWalker tw = MaryDomUtils.createTreeWalker(phrase, MaryXML.SYLLABLE); - Element syllable = (Element) MaryDomUtils.getAncestor(segment, MaryXML.SYLLABLE); - if (syllable != null) { - tw.setCurrentNode(syllable); - } else { - tw.setCurrentNode(segment); - } - Element e; - while ((e = (Element) tw.previousNode()) != null && count < RAIL_LIMIT) { - String accent = e.getAttribute("accent"); - if (!accent.equals("")) - count++; - } - return (byte) count; - } - } - - /** - * Counts the number of accented syllables until the end of the phrase. - */ - public static class AccentedSylsFromPhraseEnd implements ByteValuedFeatureProcessor { - public AccentedSylsFromPhraseEnd() { - } - - public String getName() { - return "accented_syls_from_phrase_end"; - } - - public String[] getValues() { - return ZERO_TO_NINETEEN; - } - - /** - * @param target - * the target to process - * @return the number of accented syllables since the last major break - */ - public byte process(Target target) { - Element segment = target.getMaryxmlElement(); - if (segment == null) - return 0; - Element phrase = (Element) MaryDomUtils.getAncestor(segment, MaryXML.PHRASE); - if (phrase == null) - return 0; - int count = 0; - TreeWalker tw = MaryDomUtils.createTreeWalker(phrase, MaryXML.SYLLABLE); - tw.setCurrentNode(segment); - Element e; - while ((e = (Element) tw.nextNode()) != null && count < RAIL_LIMIT) { - String accent = e.getAttribute("accent"); - if (!accent.equals("")) - count++; - } - return (byte) count; - } - } - - /** - * Counts the number of words since the start of the phrase. - */ - public static class WordsFromPhraseStart implements ByteValuedFeatureProcessor { - public WordsFromPhraseStart() { - } - - public String getName() { - return "words_from_phrase_start"; - } - - public String[] getValues() { - return ZERO_TO_NINETEEN; - } - - /** - * @param target - * the target to process - * @return the number of words since the last major break - */ - public byte process(Target target) { - Element segment = target.getMaryxmlElement(); - if (segment == null) - return 0; - Element phrase = (Element) MaryDomUtils.getAncestor(segment, MaryXML.PHRASE); - if (phrase == null) - return 0; - int count = 0; - TreeWalker tw = MaryDomUtils.createTreeWalker(phrase, MaryXML.TOKEN); - Element word = (Element) MaryDomUtils.getAncestor(segment, MaryXML.TOKEN); - if (word != null) { - tw.setCurrentNode(word); - } else { - tw.setCurrentNode(segment); - } - Element e; - while ((e = (Element) tw.previousNode()) != null && count < RAIL_LIMIT) { - // only count tokens that have a "ph" attribute: - if (e.hasAttribute("ph")) - count++; - } - return (byte) count; - } - } - - /** - * Counts the number of words until the end of the phrase. - */ - public static class WordsFromPhraseEnd implements ByteValuedFeatureProcessor { - public WordsFromPhraseEnd() { - } - - public String getName() { - return "words_from_phrase_end"; - } - - public String[] getValues() { - return ZERO_TO_NINETEEN; - } - - /** - * @param target - * the target to process - * @return the number of words until the next major break - */ - public byte process(Target target) { - Element segment = target.getMaryxmlElement(); - if (segment == null) - return 0; - Element phrase = (Element) MaryDomUtils.getAncestor(segment, MaryXML.PHRASE); - if (phrase == null) - return 0; - int count = 0; - TreeWalker tw = MaryDomUtils.createTreeWalker(phrase, MaryXML.TOKEN); - tw.setCurrentNode(segment); - Element e; - while ((e = (Element) tw.nextNode()) != null && count < RAIL_LIMIT) { - // only count tokens that have a "ph" attribute - if (e.hasAttribute("ph")) - count++; - } - return (byte) count; - } - } - - /** - * Counts the number of words since the start of the sentence. - */ - public static class WordsFromSentenceStart implements ByteValuedFeatureProcessor { - public WordsFromSentenceStart() { - } - - public String getName() { - return "words_from_sentence_start"; - } - - public String[] getValues() { - return ZERO_TO_NINETEEN; - } - - /** - * @param target - * the target to process - * @return the number of words since the beginning of the sentence - */ - public byte process(Target target) { - Element segment = target.getMaryxmlElement(); - if (segment == null) - return 0; - Element sentence = (Element) MaryDomUtils.getAncestor(segment, MaryXML.SENTENCE); - if (sentence == null) - return 0; - int count = 0; - TreeWalker tw = MaryDomUtils.createTreeWalker(sentence, MaryXML.TOKEN); - Element word = (Element) MaryDomUtils.getAncestor(segment, MaryXML.TOKEN); - if (word != null) { - tw.setCurrentNode(word); - } else { - tw.setCurrentNode(segment); - } - Element e; - while ((e = (Element) tw.previousNode()) != null && count < RAIL_LIMIT) { - // only count tokens that have a "ph" attribute: - if (e.hasAttribute("ph")) - count++; - } - return (byte) count; - } - } - - /** - * Counts the number of words until the end of the sentence. - */ - public static class WordsFromSentenceEnd implements ByteValuedFeatureProcessor { - public WordsFromSentenceEnd() { - } - - public String getName() { - return "words_from_sentence_end"; - } - - public String[] getValues() { - return ZERO_TO_NINETEEN; - } - - /** - * @param target - * the target to process - * @return the number of words until the end of the sentence - */ - public byte process(Target target) { - Element segment = target.getMaryxmlElement(); - if (segment == null) - return 0; - Element sentence = (Element) MaryDomUtils.getAncestor(segment, MaryXML.SENTENCE); - if (sentence == null) - return 0; - int count = 0; - TreeWalker tw = MaryDomUtils.createTreeWalker(sentence, MaryXML.TOKEN); - tw.setCurrentNode(segment); - Element e; - while ((e = (Element) tw.nextNode()) != null && count < RAIL_LIMIT) { - // only count tokens that have a "ph" attribute: - if (e.hasAttribute("ph")) - count++; - } - return (byte) count; - } - } - - /** - * Counts the number of phrases since the start of the sentence. - */ - public static class PhrasesFromSentenceStart implements ByteValuedFeatureProcessor { - public PhrasesFromSentenceStart() { - } - - public String getName() { - return "phrases_from_sentence_start"; - } - - public String[] getValues() { - return ZERO_TO_NINETEEN; - } - - /** - * @param target - * the target to process - * @return the number of phrases since the start of the sentence - */ - public byte process(Target target) { - Element segment = target.getMaryxmlElement(); - if (segment == null) - return 0; - Element sentence = (Element) MaryDomUtils.getAncestor(segment, MaryXML.SENTENCE); - if (sentence == null) - return 0; - int count = 0; - TreeWalker tw = MaryDomUtils.createTreeWalker(sentence, MaryXML.PHRASE); - Element phrase = (Element) MaryDomUtils.getAncestor(segment, MaryXML.PHRASE); - if (phrase != null) { - tw.setCurrentNode(phrase); - } else { - tw.setCurrentNode(segment); - } - Element e; - while ((e = (Element) tw.previousNode()) != null && count < RAIL_LIMIT) { - count++; - } - return (byte) count; - } - } - - /** - * Counts the number of phrases until the end of the sentence. - */ - public static class PhrasesFromSentenceEnd implements ByteValuedFeatureProcessor { - public PhrasesFromSentenceEnd() { - } - - public String getName() { - return "phrases_from_sentence_end"; - } - - public String[] getValues() { - return ZERO_TO_NINETEEN; - } - - /** - * @param target - * the target to process - * @return the number of phrases until the end of the sentence. - */ - public byte process(Target target) { - Element segment = target.getMaryxmlElement(); - if (segment == null) - return 0; - Element sentence = (Element) MaryDomUtils.getAncestor(segment, MaryXML.SENTENCE); - if (sentence == null) - return 0; - int count = 0; - TreeWalker tw = MaryDomUtils.createTreeWalker(sentence, MaryXML.PHRASE); - tw.setCurrentNode(segment); - Element e; - while ((e = (Element) tw.nextNode()) != null && count < RAIL_LIMIT) { - count++; - } - return (byte) count; - } - } - - /** - * Counts the number of syllables since the last accent in the current phrase. - */ - public static class SylsFromPrevAccent implements ByteValuedFeatureProcessor { - public SylsFromPrevAccent() { - } - - public String getName() { - return "syls_from_prev_accent"; - } - - public String[] getValues() { - return ZERO_TO_NINETEEN; - } - - /** - * @param target - * the target to process - * @return the number of syllables since the last accent - */ - public byte process(Target target) { - Element segment = target.getMaryxmlElement(); - if (segment == null) - return 0; - Element phrase = (Element) MaryDomUtils.getAncestor(segment, MaryXML.PHRASE); - if (phrase == null) - return 0; - int count = 0; - TreeWalker tw = MaryDomUtils.createTreeWalker(phrase, MaryXML.SYLLABLE); - Element syllable = (Element) MaryDomUtils.getAncestor(segment, MaryXML.SYLLABLE); - if (syllable != null) { - tw.setCurrentNode(syllable); - } else { - tw.setCurrentNode(segment); - } - Element e; - while ((e = (Element) tw.previousNode()) != null && count < RAIL_LIMIT) { - count++; - String accent = e.getAttribute("accent"); - if (!accent.equals("")) - break; - } - return (byte) count; - } - } - - /** - * Counts the number of syllables until the next accent in the current phrase. - */ - public static class SylsToNextAccent implements ByteValuedFeatureProcessor { - public SylsToNextAccent() { - } - - public String getName() { - return "syls_to_next_accent"; - } - - public String[] getValues() { - return ZERO_TO_NINETEEN; - } - - /** - * @param target - * the target to process - * @return the number of syllables until the next accent - */ - public byte process(Target target) { - Element segment = target.getMaryxmlElement(); - if (segment == null) - return 0; - Element phrase = (Element) MaryDomUtils.getAncestor(segment, MaryXML.PHRASE); - if (phrase == null) - return 0; - int count = 0; - TreeWalker tw = MaryDomUtils.createTreeWalker(phrase, MaryXML.SYLLABLE); - tw.setCurrentNode(segment); - Element e; - while ((e = (Element) tw.nextNode()) != null && count < RAIL_LIMIT) { - count++; - String accent = e.getAttribute("accent"); - if (!accent.equals("")) - break; - } - return (byte) count; - } - } - - /** - * Counts the number of syllables since the last stressed syllable in the current phrase. - */ - public static class SylsFromPrevStressed implements ByteValuedFeatureProcessor { - public SylsFromPrevStressed() { - } - - public String getName() { - return "syls_from_prev_stressed"; - } - - public String[] getValues() { - return ZERO_TO_NINETEEN; - } - - /** - * @param target - * the target to process - * @return the number of syllables since the last accent - */ - public byte process(Target target) { - Element segment = target.getMaryxmlElement(); - if (segment == null) - return 0; - Element phrase = (Element) MaryDomUtils.getAncestor(segment, MaryXML.PHRASE); - if (phrase == null) - return 0; - int count = 0; - TreeWalker tw = MaryDomUtils.createTreeWalker(phrase, MaryXML.SYLLABLE); - Element syllable = (Element) MaryDomUtils.getAncestor(segment, MaryXML.SYLLABLE); - if (syllable != null) { - tw.setCurrentNode(syllable); - } else { - tw.setCurrentNode(segment); - } - Element e; - while ((e = (Element) tw.previousNode()) != null && count < RAIL_LIMIT) { - count++; - String stress = e.getAttribute("stress"); - if (stress.equals("1")) - break; - } - return (byte) count; - } - } - - /** - * Counts the number of syllables until the next stressed syllable in the current phrase. - */ - public static class SylsToNextStressed implements ByteValuedFeatureProcessor { - public SylsToNextStressed() { - } - - public String getName() { - return "syls_to_next_stressed"; - } - - public String[] getValues() { - return ZERO_TO_NINETEEN; - } - - /** - * @param target - * the target to process - * @return the number of syllables until the next stressed syllable - */ - public byte process(Target target) { - Element segment = target.getMaryxmlElement(); - if (segment == null) - return 0; - Element phrase = (Element) MaryDomUtils.getAncestor(segment, MaryXML.PHRASE); - if (phrase == null) - return 0; - int count = 0; - TreeWalker tw = MaryDomUtils.createTreeWalker(phrase, MaryXML.SYLLABLE); - tw.setCurrentNode(segment); - Element e; - while ((e = (Element) tw.nextNode()) != null && count < RAIL_LIMIT) { - count++; - String stress = e.getAttribute("stress"); - if (stress.equals("1")) - break; - } - return (byte) count; - } - } - - /** - * Determines the word punctuation. - */ - public static class WordPunc implements ByteValuedFeatureProcessor { - protected String name; - protected TargetElementNavigator navigator; - protected ByteStringTranslator values; - - /** - * @param name - * name of this feature processor - * @param wordNavigator - * a navigator which returns a word for a target. This navigator decides the word for which the punctuation - * will be computed. - */ - public WordPunc(String name, TargetElementNavigator wordNavigator) { - this.name = name; - this.navigator = wordNavigator; - this.values = new ByteStringTranslator(new String[] { "0", ".", ",", ";", ":", "(", ")", "?", "!", "\"" }); - } - - public String getName() { - return name; - } - - public String[] getValues() { - return values.getStringValues(); - } - - public byte process(Target target) { - Element word = navigator.getElement(target); - if (word == null) - return 0; - Element sentence = (Element) MaryDomUtils.getAncestor(word, MaryXML.SENTENCE); - if (sentence == null) - return 0; - TreeWalker tw = MaryDomUtils.createTreeWalker(sentence, MaryXML.TOKEN, MaryXML.BOUNDARY); - tw.setCurrentNode(word); - Element next = (Element) tw.nextNode(); - if (next == null || !next.getTagName().equals(MaryXML.TOKEN) || next.hasAttribute("ph")) - return 0; - String text = MaryDomUtils.tokenText(next); - if (values.contains(text)) { - return values.get(text); - } - // unknown or no punctuation: return "0" - return values.get("0"); - } - } - - /** - * Determines the next word punctuation in the sentence. - */ - public static class NextPunctuation extends WordPunc { - public NextPunctuation() { - super("next_punctuation", new WordNavigator()); - } - - public byte process(Target target) { - Element word = navigator.getElement(target); - if (word == null) - return 0; - Element sentence = (Element) MaryDomUtils.getAncestor(word, MaryXML.SENTENCE); - if (sentence == null) - return 0; - TreeWalker tw = MaryDomUtils.createTreeWalker(sentence, MaryXML.TOKEN); - tw.setCurrentNode(word); - Element e; - while ((e = (Element) tw.nextNode()) != null) { - if (e.hasAttribute("ph")) // a word - continue; - // potentially a punctuation - String text = MaryDomUtils.tokenText(e); - if (values.contains(text)) { - return values.get(text); - } - } - // no next punctuation: return "0" - return values.get("0"); - } - } - - /** - * Determines the previous word punctuation in the sentence. - */ - public static class PrevPunctuation extends WordPunc { - public PrevPunctuation() { - super("prev_punctuation", new WordNavigator()); - } - - public byte process(Target target) { - Element word = navigator.getElement(target); - if (word == null) - return 0; - Element sentence = (Element) MaryDomUtils.getAncestor(word, MaryXML.SENTENCE); - if (sentence == null) - return 0; - TreeWalker tw = MaryDomUtils.createTreeWalker(sentence, MaryXML.TOKEN); - tw.setCurrentNode(word); - Element e; - while ((e = (Element) tw.previousNode()) != null) { - if (e.hasAttribute("ph")) // a word - continue; - // potentially a punctuation - String text = MaryDomUtils.tokenText(e); - if (values.contains(text)) { - return values.get(text); - } - } - // no next punctuation: return "0" - return values.get("0"); - } - } - - /** - * Determines the distance in words to the next word punctuation in the sentence. - */ - public static class WordsToNextPunctuation extends WordPunc { - public WordsToNextPunctuation() { - super("words_to_next_punctuation", new WordNavigator()); - } - - public String[] getValues() { - return ZERO_TO_NINETEEN; - } - - public byte process(Target target) { - Element word = navigator.getElement(target); - if (word == null) - return 0; - Element sentence = (Element) MaryDomUtils.getAncestor(word, MaryXML.SENTENCE); - if (sentence == null) - return 0; - TreeWalker tw = MaryDomUtils.createTreeWalker(sentence, MaryXML.TOKEN); - tw.setCurrentNode(word); - Element e; - int count = 0; - while ((e = (Element) tw.nextNode()) != null && count < RAIL_LIMIT) { - count++; - if (e.hasAttribute("ph")) // a word - continue; - // potentially a punctuation - String text = MaryDomUtils.tokenText(e); - if (values.contains(text)) { - break; - } - } - // found punctuation or end of sentence: - return (byte) count; - } - } - - /** - * Determines the distance in words from the previous word punctuation in the sentence. - */ - public static class WordsFromPrevPunctuation extends WordPunc { - public WordsFromPrevPunctuation() { - super("words_from_prev_punctuation", new WordNavigator()); - } - - public String[] getValues() { - return ZERO_TO_NINETEEN; - } - - public byte process(Target target) { - Element word = navigator.getElement(target); - if (word == null) - return 0; - Element sentence = (Element) MaryDomUtils.getAncestor(word, MaryXML.SENTENCE); - if (sentence == null) - return 0; - TreeWalker tw = MaryDomUtils.createTreeWalker(sentence, MaryXML.TOKEN); - tw.setCurrentNode(word); - Element e; - int count = 0; - while ((e = (Element) tw.previousNode()) != null && count < RAIL_LIMIT) { - count++; - if (e.hasAttribute("ph")) // a word - continue; - // potentially a punctuation - String text = MaryDomUtils.tokenText(e); - if (values.contains(text)) { - break; - } - } - // found punctuation or start of sentence: - return (byte) count; - } - } - - /** - * Determine the prosodic property of a target - * - * @author Anna Hunecke - * - */ - public static class Selection_Prosody implements ByteValuedFeatureProcessor { - - protected TargetElementNavigator navigator; - protected ByteStringTranslator values = new ByteStringTranslator(new String[] { "0", "stressed", "pre-nuclear", - "nuclear", "finalHigh", "finalLow", "final" }); - private Set lowEndtones = new HashSet(Arrays.asList(new String[] { "L-", "L-%", "L-L%" })); - private Set highEndtones = new HashSet(Arrays.asList(new String[] { "H-", "!H-", "H-%", "H-L%", "!H-%", - "H-^H%", "!H-^H%", "L-H%", "H-H%" })); - - public Selection_Prosody(TargetElementNavigator syllableNavigator) { - this.navigator = syllableNavigator; - } - - public String getName() { - return "selection_prosody"; - } - - public String[] getValues() { - return values.getStringValues(); - } - - /** - * Determine the prosodic property of the target - * - * @param target - * the target - * @return 0 - unstressed, 1 - stressed, 2 - pre-nuclear accent 3 - nuclear accent, 4 - phrase final high, 5 - phrase - * final low, 6 - phrase final (with unknown high/low status). - */ - public byte process(Target target) { - // first find out if syllable is stressed - Element syllable = navigator.getElement(target); - if (syllable == null) - return (byte) 0; - boolean stressed = false; - if (syllable.getAttribute("stress").equals("1")) { - stressed = true; - } - // find out if we have an accent - boolean accented = syllable.hasAttribute("accent"); - boolean nuclear = true; // relevant only if accented == true - // find out the position of the target - boolean phraseFinal = false; - String endtone = null; - Element sentence = (Element) MaryDomUtils.getAncestor(syllable, MaryXML.SENTENCE); - if (sentence == null) - return 0; - TreeWalker tw = MaryDomUtils.createTreeWalker(sentence, MaryXML.SYLLABLE, MaryXML.BOUNDARY); - tw.setCurrentNode(syllable); - Element e = (Element) tw.nextNode(); - if (e != null) { - if (e.getTagName().equals(MaryXML.BOUNDARY)) { - phraseFinal = true; - endtone = e.getAttribute("tone"); - } - if (accented) { // look forward for any accent - while (e != null) { - if (e.getTagName().equals(MaryXML.SYLLABLE) && e.hasAttribute("accent")) { - nuclear = false; - break; - } - e = (Element) tw.nextNode(); - } - } - } - // Now, we know: - // stressed or not - // accented or not - // if accented, nuclear or not - // if final, the endtone - - if (accented) { - if (nuclear) { - return values.get("nuclear"); - } else { - return values.get("pre-nuclear"); - } - } else if (phraseFinal) { - if (endtone != null && highEndtones.contains(endtone)) { - return values.get("finalHigh"); - } else if (endtone != null && lowEndtones.contains(endtone)) { - return values.get("finalLow"); - } else { - return values.get("final"); - } - } else if (stressed) { - return values.get("stressed"); - } - return (byte) 0;// return unstressed - } - } - - /** - * Returns the duration of the given segment, in seconds. - */ - public static class UnitDuration implements ContinuousFeatureProcessor { - public String getName() { - return "unit_duration"; - } - - public float process(Target target) { - if (target instanceof DiphoneTarget) { - DiphoneTarget diphone = (DiphoneTarget) target; - return process(diphone.left) + process(diphone.right); - } - Element seg = target.getMaryxmlElement(); - if (seg == null) { - return 0; - } - float phoneDuration = 0; - String sDur; - if (seg.getTagName().equals(MaryXML.PHONE)) - sDur = seg.getAttribute("d"); - else { - assert seg.getTagName().equals(MaryXML.BOUNDARY) : "segment should be a phone or a boundary, but is a " - + seg.getTagName(); - sDur = seg.getAttribute("duration"); - } - if (sDur.equals("")) { - return 0; - } - try { - // parse duration string, and convert from milliseconds into seconds: - phoneDuration = Float.parseFloat(sDur) * 0.001f; - } catch (NumberFormatException nfe) { - } - if (target instanceof HalfPhoneTarget) - return phoneDuration / 2; - return phoneDuration; - } - } - - /** - * Calculates the log of the fundamental frequency in the middle of a unit segment. This processor should be used by target - * items only -- for unit features during voice building, the actual measured values should be used. - */ - public static class UnitLogF0 implements ContinuousFeatureProcessor { - public String getName() { - return "unit_logf0"; - } - - public float process(Target target) { - return process(target, false); - } - - /** - * Compute log f0 and log f0 delta for the given target. - * - * @param target - * @param delta - * if true, return the delta, i.e. the logF0 slope; if false, return the log f0 value itself. - * @return - */ - protected float process(Target target, boolean delta) { - // Note: all variables in this method with "f0" in their name - // actually represent log f0 values. - if (target instanceof DiphoneTarget) { - DiphoneTarget diphone = (DiphoneTarget) target; - return (process(diphone.left) + process(diphone.right)) / 2; - } - // Idea: find the closest f0 targets in the current syllable, left and right of our middle; - // linearly interpolate between them to find the value in the middle of this unit. - Element seg = target.getMaryxmlElement(); - if (seg == null) { - return 0; - } - if (!seg.getTagName().equals(MaryXML.PHONE)) { - return 0; - } - // get mid position of segment wrt phone start (phone start = 0, phone end = phone duration) - float mid; - float phoneDuration = getDuration(seg); - if (target instanceof HalfPhoneTarget) { - if (((HalfPhoneTarget) target).isLeftHalf()) { - mid = .25f; - } else { - mid = .75f; - } - } else { // phone target - mid = .5f; - } - - // Now mid is the middle of the unit relative to the phone start, in percent - Float lastPos = null; // position relative to mid, in milliseconds (negative) - float lastF0 = 0; - Float nextPos = null; // position relative to mid, in milliseconds - float nextF0 = 0; - Float[] f0values = getLogF0Values(seg); - - assert f0values != null; - // values are position, f0, position, f0, etc.; - // position is in percent of phone duration between 0 and 1, f0 is in Hz - for (int i = 0; i < f0values.length; i += 2) { - float pos = f0values[i]; - if (pos <= mid) { - lastPos = (pos - mid) * phoneDuration; // negative or zero - lastF0 = f0values[i + 1]; - } else if (pos > mid) { - nextPos = (pos - mid) * phoneDuration; // positive - nextF0 = f0values[i + 1]; - break; // no point looking further to the right - } - } - if (lastPos == null) { // need to look to the left - float msBack = -mid * phoneDuration; - Element e = seg; - - // get all phone units in the same phrase - Element phraseElement = (Element) MaryDomUtils.getAncestor(seg, MaryXML.PHRASE); - TreeWalker tw = MaryDomUtils.createTreeWalker(seg.getOwnerDocument(), phraseElement, MaryXML.PHONE); - Element en; - while ((en = (Element) tw.nextNode()) != null) { - if (en == seg) { - break; - } - } - - while ((e = (Element) tw.previousNode()) != null) { - float dur = getDuration(e); - f0values = getLogF0Values(e); - if (f0values.length == 0) { - msBack -= dur; - continue; - } - assert f0values.length > 1; - float pos = f0values[f0values.length - 2]; - lastPos = msBack - (1 - pos) * dur; - lastF0 = f0values[f0values.length - 1]; - break; - } - } - - if (nextPos == null) { // need to look to the right - float msForward = (1 - mid) * phoneDuration; - Element e = seg; - - // get all phone units in the same phrase - Element phraseElement = (Element) MaryDomUtils.getAncestor(seg, MaryXML.PHRASE); - TreeWalker tw = MaryDomUtils.createTreeWalker(seg.getOwnerDocument(), phraseElement, MaryXML.PHONE); - Element en; - while ((en = (Element) tw.nextNode()) != null) { - if (en == seg) { - break; - } - } - - while ((e = (Element) tw.nextNode()) != null) { - float dur = getDuration(e); - f0values = getLogF0Values(e); - if (f0values.length == 0) { - msForward += dur; - continue; - } - assert f0values.length > 1; - float pos = f0values[0]; - nextPos = msForward + pos * dur; - nextF0 = f0values[1]; - break; - } - } - - if (lastPos == null && nextPos == null) { - // no info - return 0; - } else if (lastPos == null) { - // have only nextF0; - if (delta) - return 0; - else - return nextF0; - } else if (nextPos == null) { - // have only lastF0 - if (delta) - return 0; - else - return lastF0; - } - assert lastPos <= 0 && 0 <= nextPos : "unexpected: lastPos=" + lastPos + ", nextPos=" + nextPos; - // build a linear function (f(x) = slope*x+intersectionYAxis) - float f0; - float slope; - if (lastPos - nextPos == 0) { - f0 = (lastF0 + nextF0) / 2; - slope = 0; - } else { - slope = (nextF0 - lastF0) / (nextPos - lastPos); - // calculate the pitch - f0 = lastF0 + slope * (-lastPos); - } - assert !Float.isNaN(f0) : "f0 is not a number"; - assert lastF0 <= f0 && nextF0 >= f0 || lastF0 >= f0 && nextF0 <= f0 : "f0 should be between last and next values"; - - if (delta) - return slope; - else - return f0; - } - - private Float[] getLogF0Values(Element ph) { - String mbrTargets = ph.getAttribute("f0"); - if (mbrTargets.equals("")) { - return new Float[0]; - } - ArrayList values = new ArrayList(); - try { - // mbrTargets contains one or more pairs of numbers, - // either enclosed by (a,b) or just separated by whitespace. - StringTokenizer st = new StringTokenizer(mbrTargets, " (,)"); - while (st.hasMoreTokens()) { - String posString = ""; - while (st.hasMoreTokens() && posString.equals("")) - posString = st.nextToken(); - String f0String = ""; - while (st.hasMoreTokens() && f0String.equals("")) - f0String = st.nextToken(); - - float pos = Float.parseFloat(posString) * 0.01f; - assert 0 <= pos && pos <= 1 : "invalid position:" + pos + " (pos string was '" + posString - + "' coming from '" + mbrTargets + "')"; - float f0 = Float.parseFloat(f0String); - float logF0 = (float) Math.log(f0); - values.add(pos); - values.add(logF0); - } - } catch (Exception e) { - return new Float[0]; - } - return values.toArray(new Float[0]); - } - - private float getDuration(Element ph) { - float phoneDuration = 0; - String sDur = ph.getAttribute("d"); - if (!sDur.equals("")) { - try { - phoneDuration = Float.parseFloat(sDur); - } catch (NumberFormatException nfe) { - } - } - return phoneDuration; - } - } - - /** - * Calculates the slope of a linear approximation of the fundamental frequency, in the log domain. The slope is computed by - * linearly connecting the two log f0 values closest to the middle of the unit segment. This processor should be used by - * target items only -- for unit features during voice building, the actual measured values should be used. - */ - public static class UnitLogF0Delta extends UnitLogF0 { - @Override - public String getName() { - return "unit_logf0delta"; - } - - public float process(Target target) { - return process(target, true); - } - } - - /** - * Returns the value of the given feature for the given segment. - */ - public static class GenericContinuousFeature implements ContinuousFeatureProcessor { - private String name; - private String attributeName; - - public GenericContinuousFeature(String featureName, String attributeName) { - this.name = featureName; - this.attributeName = attributeName; - } - - public String getName() { - return name; - } - - public float process(Target target) { - if (target instanceof DiphoneTarget) { - DiphoneTarget diphone = (DiphoneTarget) target; - // return mean of left and right costs: - return (process(diphone.left) + process(diphone.right)) / 2.0f; - } - Element seg = target.getMaryxmlElement(); - if (seg == null) { - return 0; - } - float value = 0; - String valueString; - if (seg.getTagName().equals(MaryXML.PHONE)) { - valueString = seg.getAttribute(attributeName); - } else { - assert seg.getTagName().equals(MaryXML.BOUNDARY) : "segment should be a phone or a boundary, but is a " - + seg.getTagName(); - valueString = seg.getAttribute(attributeName); - } - if (valueString.equals("")) { - return 0; - } - try { - value = Float.parseFloat(valueString); - } catch (NumberFormatException nfe) { - return 0; - } - return value; - } - } - -} diff --git a/marytts-unitselection/src/main/java/marytts/features/MaryLanguageFeatureProcessors.java b/marytts-unitselection/src/main/java/marytts/features/MaryLanguageFeatureProcessors.java deleted file mode 100644 index 5813f90c61..0000000000 --- a/marytts-unitselection/src/main/java/marytts/features/MaryLanguageFeatureProcessors.java +++ /dev/null @@ -1,468 +0,0 @@ -/** - * Portions Copyright 2006 DFKI GmbH. - * Portions Copyright 2001 Sun Microsystems, Inc. - * Portions Copyright 1999-2001 Language Technologies Institute, - * Carnegie Mellon University. - * All Rights Reserved. Use is subject to license terms. - * - * Permission is hereby granted, free of charge, to use and distribute - * this software and its documentation without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of this work, and to - * permit persons to whom this work is furnished to do so, subject to - * the following conditions: - * - * 1. The code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * 2. Any modifications must be clearly marked as such. - * 3. Original authors' names are not deleted. - * 4. The authors' names are not used to endorse or promote products - * derived from this software without specific prior written - * permission. - * - * DFKI GMBH AND THE CONTRIBUTORS TO THIS WORK DISCLAIM ALL WARRANTIES WITH - * REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL DFKI GMBH NOR THE - * CONTRIBUTORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL - * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR - * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS - * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF - * THIS SOFTWARE. - */ -package marytts.features; - -import java.io.InputStream; -import java.util.Map; - -import marytts.datatypes.MaryXML; -import marytts.fst.FSTLookup; -import marytts.modules.phonemiser.AllophoneSet; -import marytts.util.dom.MaryDomUtils; -import marytts.util.string.ByteStringTranslator; - -import org.w3c.dom.Element; - -/** - * Provides the set of feature processors that are used by this language as part of the CART processing. - */ -public class MaryLanguageFeatureProcessors extends MaryGenericFeatureProcessors { - - // no instances - private MaryLanguageFeatureProcessors() { - } - - /** - * The phone symbol for the given target. - * - * @author Marc Schröder - * - */ - public static class Phone implements ByteValuedFeatureProcessor { - protected String name; - protected ByteStringTranslator values; - protected String pauseSymbol; - protected TargetElementNavigator navigator; - - /** - * Initialise a phone feature processor. - * - * @param name - * the name of the feature - * @param possibleValues - * the list of possible phone values for the phonetic alphabet used, plus the value "0"=n/a. - * @param segmentNavigator - * a navigator returning a segment with respect to the target. - */ - public Phone(String name, String[] possibleValues, String pauseSymbol, TargetElementNavigator segmentNavigator) { - this.name = name; - this.values = new ByteStringTranslator(possibleValues); - this.pauseSymbol = pauseSymbol; - this.navigator = segmentNavigator; - } - - public String getName() { - return name; - } - - public String[] getValues() { - return values.getStringValues(); - } - - public byte process(Target target) { - Element segment = navigator.getElement(target); - if (segment == null) - return values.get(pauseSymbol); - if (!segment.getTagName().equals(MaryXML.PHONE)) - return values.get(pauseSymbol); - String ph = segment.getAttribute("p"); - if (!values.contains(ph)) - return values.get("0"); - return values.get(ph); - } - - public String getPauseSymbol() { - return pauseSymbol; - } - } - - /** - * The unit name for the given half phone target. - * - * @author Marc Schröder - * - */ - public static class HalfPhoneUnitName implements ByteValuedFeatureProcessor { - protected String name; - protected ByteStringTranslator values; - protected String pauseSymbol; - - /** - * Initialise a UnitName feature processor. - * - * @param name - * the name of the feature - * @param phoneset - * the phonetic alphabet used - * @param segmentNavigator - * a navigator returning a segment with respect to the target. - */ - public HalfPhoneUnitName(String[] possiblePhonemes, String pauseSymbol) { - this.name = "halfphone_unitname"; - this.pauseSymbol = pauseSymbol; - String[] possibleValues = new String[2 * possiblePhonemes.length + 1]; - possibleValues[0] = "0"; // the "n/a" value - for (int i = 0; i < possiblePhonemes.length; i++) { - possibleValues[2 * i + 1] = possiblePhonemes[i] + "_L"; - possibleValues[2 * i + 2] = possiblePhonemes[i] + "_R"; - } - this.values = new ByteStringTranslator(possibleValues); - } - - public String getName() { - return name; - } - - public String[] getValues() { - return values.getStringValues(); - } - - public byte process(Target target) { - if (!(target instanceof HalfPhoneTarget)) - return 0; - HalfPhoneTarget hpTarget = (HalfPhoneTarget) target; - Element segment = target.getMaryxmlElement(); - String phoneLabel; - if (segment == null) { - phoneLabel = pauseSymbol; - } else if (!segment.getTagName().equals(MaryXML.PHONE)) { - phoneLabel = pauseSymbol; - } else { - phoneLabel = segment.getAttribute("p"); - } - if (phoneLabel.equals("")) - return values.get("0"); - String unitLabel = phoneLabel + (hpTarget.isLeftHalf() ? "_L" : "_R"); - return values.get(unitLabel); - } - } - - /** - * A parametrisable class which can retrieve all sorts of phone features, given a phone set. - * - * @author Marc Schröder - * - */ - public static class PhoneFeature implements ByteValuedFeatureProcessor { - protected AllophoneSet phoneSet; - protected String name; - protected String phonesetQuery; - protected ByteStringTranslator values; - protected String pauseSymbol; - protected TargetElementNavigator navigator; - - public PhoneFeature(AllophoneSet phoneSet, String name, String phonesetQuery, String[] possibleValues, - String pauseSymbol, TargetElementNavigator segmentNavigator) { - this.phoneSet = phoneSet; - this.name = name; - this.phonesetQuery = phonesetQuery; - this.values = new ByteStringTranslator(possibleValues); - this.navigator = segmentNavigator; - } - - public String getName() { - return name; - } - - public String[] getValues() { - return values.getStringValues(); - } - - public byte process(Target target) { - Element segment = navigator.getElement(target); - if (segment == null) - return values.get("0"); - String ph; - if (!segment.getTagName().equals(MaryXML.PHONE)) { - ph = pauseSymbol; - } else { - ph = segment.getAttribute("p"); - } - String value = phoneSet.getPhoneFeature(ph, phonesetQuery); - if (value == null) - return values.get("0"); - return values.get(value); - } - } - - /** - * Returns the part-of-speech. - */ - public static class Pos implements ByteValuedFeatureProcessor { - private ByteStringTranslator values; - private TargetElementNavigator navigator; - private String name; - - public String getName() { - return this.name; - } - - public String[] getValues() { - return values.getStringValues(); - } - - public Pos(String[] posValues) { - this.values = new ByteStringTranslator(posValues); - this.navigator = new WordNavigator(); - this.name = "pos"; - } - - public Pos(String aName, String[] posValues, TargetElementNavigator wordNavi) { - this.values = new ByteStringTranslator(posValues); - this.navigator = wordNavi; - this.name = aName; - } - - /** - * @param item - * the item to process - * @return a guess at the part-of-speech for the item - */ - public byte process(Target target) { - Element word = navigator.getElement(target); - if (word == null) - return values.get("0"); - String pos = word.getAttribute("pos"); - if (pos == null) - return values.get("0"); - pos = pos.trim(); - if (values.contains(pos)) - return values.get(pos); - return values.get("0"); - } - } - - /** - * Returns generalised part-of-speech. - */ - public static class Gpos implements ByteValuedFeatureProcessor { - private Map posConverter; - private ByteStringTranslator values; - private TargetElementNavigator navigator; - - public String getName() { - return "gpos"; - } - - public String[] getValues() { - return values.getStringValues(); - } - - public Gpos(Map posConverter) { - this.posConverter = posConverter; - this.values = new ByteStringTranslator(new String[] { "0", "in", // Preposition or subordinating conjunction - "to", // to - "det", // determiner - "md", // modal - "cc", // coordinating conjunction - "wp", // w-pronouns - "pps", // possive pronouns - "aux", // auxiliary verbs - "punc", // punctuation - "content" // content words - }); - this.navigator = new WordNavigator(); - } - - /** - * Performs some processing on the given item. - * - * @param item - * the item to process - * @return a guess at the part-of-speech for the item - */ - public byte process(Target target) { - Element word = navigator.getElement(target); - if (word == null) - return values.get("0"); - String pos = word.getAttribute("pos"); - if (pos == null) - return values.get("0"); - pos = pos.trim(); - if (posConverter.containsKey(pos)) { - pos = (String) posConverter.get(pos); - } - if (!values.contains(pos)) - return values.get("0"); - return values.get(pos); - } - } - - /** - * Checks for onset coda This is a feature processor. A feature processor takes an item, performs some sort of processing on - * the item and returns an object. - */ - public static class SegOnsetCoda implements ByteValuedFeatureProcessor { - protected ByteStringTranslator values; - private AllophoneSet phoneSet; - - public SegOnsetCoda(AllophoneSet phoneSet) { - this.phoneSet = phoneSet; - this.values = new ByteStringTranslator(new String[] { "0", "onset", "coda" }); - } - - public String getName() { - return "onsetcoda"; - } - - public String[] getValues() { - return values.getStringValues(); - } - - public byte process(Target target) { - Element s = target.getMaryxmlElement(); - if (s == null) { - return 0; - } - if (!s.getTagName().equals(MaryXML.PHONE)) - return 0; - - while ((s = MaryDomUtils.getNextSiblingElement(s)) != null) { - String ph = s.getAttribute("p"); - if ("+".equals(phoneSet.getPhoneFeature(ph, "vc"))) { - return values.get("onset"); - } - } - return values.get("coda"); - } - } - - /** - * The phone class for the given target. - * - * @author Anna Hunecke - * - */ - public static class Selection_PhoneClass implements ByteValuedFeatureProcessor { - protected String name; - protected Map phones2Classes; - protected ByteStringTranslator values; - protected TargetElementNavigator navigator; - - /** - * Initialise the feature processor. - * - * @param phones2Classes - * the mapping of phones to their classes - * @param classes - * the available phone classes - * @param segmentNavigator - * a navigator returning a segment with respect to the target. - */ - public Selection_PhoneClass(Map phones2Classes, String[] classes, TargetElementNavigator segmentNavigator) { - this.name = "selection_next_phone_class"; - this.phones2Classes = phones2Classes; - this.values = new ByteStringTranslator(classes); - this.navigator = segmentNavigator; - } - - public String getName() { - return name; - } - - public String[] getValues() { - return values.getStringValues(); - } - - /** - * Give back the phone class of the target - * - * @param target - * @return the phone class of the target - */ - public byte process(Target target) { - Element segment = navigator.getElement(target); - if (segment == null) - return values.get("0"); - if (!segment.getTagName().equals(MaryXML.PHONE)) - return 0; - String ph = segment.getAttribute("p"); - String phoneClass = phones2Classes.get(ph); - if (phoneClass == null) { - return values.get("0"); - } - return values.get(phoneClass); - } - } - - public static class WordFrequency implements ByteValuedFeatureProcessor { - protected TargetElementNavigator navigator; - protected ByteStringTranslator values; - protected FSTLookup wordFrequencies; - - public WordFrequency(InputStream inStream, String identifier, String encoding) { - this.navigator = new WordNavigator(); - try { - if (inStream != null) - this.wordFrequencies = new FSTLookup(inStream, identifier, encoding); - else - this.wordFrequencies = null; - } catch (Exception e) { - throw new RuntimeException(e); - } - this.values = new ByteStringTranslator(new String[] { "0", "1", "2", "3", "4", "5", "6", "7", "8", "9" }); - } - - public String getName() { - return "word_frequency"; - } - - public String[] getValues() { - return values.getStringValues(); - } - - /** - * Performs some processing on the given item. - * - * @param target - * the target to process - * @return the frequency of the current word, on a ten-point scale from 0=unknown=very rare to 9=very frequent. - */ - public byte process(Target target) { - Element word = navigator.getElement(target); - if (word == null) - return (byte) 0; - String wordString = MaryDomUtils.tokenText(word); - if (wordFrequencies != null) { - String[] result = wordFrequencies.lookup(wordString); - if (result.length > 0) { - String freq = result[0]; - if (values.contains(freq)) - return values.get(freq); - } - - } - return (byte) 0; // unknown word - } - } - -} diff --git a/marytts-unitselection/src/main/java/marytts/features/ShortValuedFeatureProcessor.java b/marytts-unitselection/src/main/java/marytts/features/ShortValuedFeatureProcessor.java deleted file mode 100644 index bd36a1048b..0000000000 --- a/marytts-unitselection/src/main/java/marytts/features/ShortValuedFeatureProcessor.java +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Portions Copyright 2006 DFKI GmbH. - * Portions Copyright 2001 Sun Microsystems, Inc. - * Portions Copyright 1999-2001 Language Technologies Institute, - * Carnegie Mellon University. - * All Rights Reserved. Use is subject to license terms. - * - * Permission is hereby granted, free of charge, to use and distribute - * this software and its documentation without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of this work, and to - * permit persons to whom this work is furnished to do so, subject to - * the following conditions: - * - * 1. The code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * 2. Any modifications must be clearly marked as such. - * 3. Original authors' names are not deleted. - * 4. The authors' names are not used to endorse or promote products - * derived from this software without specific prior written - * permission. - * - * DFKI GMBH AND THE CONTRIBUTORS TO THIS WORK DISCLAIM ALL WARRANTIES WITH - * REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL DFKI GMBH NOR THE - * CONTRIBUTORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL - * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR - * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS - * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF - * THIS SOFTWARE. - */ -package marytts.features; - - -/** - * Performs a specific type of processing on an item and returns an object. - */ -public interface ShortValuedFeatureProcessor extends MaryFeatureProcessor { - /** - * List the possible values of the feature processor, as clear-text values. Short values as returned by process() can be - * translated into their string equivalent by using the short value as an index in the String[] returned. - * - * @return an array containing the possible return values of this feature processor, in String representation. - */ - public String[] getValues(); - - public short process(Target target); -} diff --git a/marytts-unitselection/src/main/java/marytts/features/Target.java b/marytts-unitselection/src/main/java/marytts/features/Target.java deleted file mode 100644 index 25486764dd..0000000000 --- a/marytts-unitselection/src/main/java/marytts/features/Target.java +++ /dev/null @@ -1,191 +0,0 @@ -/** - * Copyright 2006 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.features; - -import marytts.datatypes.MaryXML; -import marytts.features.FeatureVector; -import marytts.features.MaryGenericFeatureProcessors; -import marytts.modules.phonemiser.Allophone; -import marytts.modules.phonemiser.AllophoneSet; -import marytts.modules.synthesis.Voice; -import marytts.util.MaryRuntimeUtils; -import marytts.util.dom.MaryDomUtils; - -import org.w3c.dom.Element; -import org.w3c.dom.Node; -import org.w3c.dom.UserDataHandler; - -/** - * A representation of a target representing the ideal properties of a unit in a target utterance. - * - * @author Marc Schröder - * - */ -public class Target { - protected String name; - protected Element maryxmlElement; - - protected FeatureVector featureVector = null; - - protected float duration = -1; - protected float f0 = -1; - protected int isSilence = -1; - - /** - * Create a target associated to the given element in the MaryXML tree. - * - * @param name - * a name for the target, which may or may not coincide with the segment name. - * @param maryxmlElement - * the phone or boundary element in the MaryXML tree to be associated with this target. - */ - public Target(String name, Element maryxmlElement) { - this.name = name; - this.maryxmlElement = maryxmlElement; - } - - public Element getMaryxmlElement() { - return maryxmlElement; - } - - public String getName() { - return name; - } - - public FeatureVector getFeatureVector() { - return featureVector; - } - - public void setFeatureVector(FeatureVector featureVector) { - this.featureVector = featureVector; - } - - public float getTargetDurationInSeconds() { - if (duration != -1) { - return duration; - } else { - if (maryxmlElement == null) - return 0; - // throw new NullPointerException("Target "+name+" does not have a maryxml element."); - duration = new MaryGenericFeatureProcessors.UnitDuration().process(this); - return duration; - } - } - - /** - * adapted from {@link MaryGenericFeatureProcessors.UnitDuration#process} - * - * @param newDuration - */ - public void setTargetDurationInSeconds(float newDuration) { - if (maryxmlElement != null) { - if (maryxmlElement.getTagName().equals(MaryXML.PHONE)) { - maryxmlElement.setAttribute("d", Float.toString(newDuration)); - } else { - assert maryxmlElement.getTagName().equals(MaryXML.BOUNDARY) : "segment should be a phone or a boundary, but is a " - + maryxmlElement.getTagName(); - maryxmlElement.setAttribute("duration", Float.toString(newDuration)); - } - } - } - - public float getTargetF0InHz() { - if (f0 != -1) { - return f0; - } else { - if (maryxmlElement == null) - throw new NullPointerException("Target " + name + " does not have a maryxml element."); - float logf0 = new MaryGenericFeatureProcessors.UnitLogF0().process(this); - if (logf0 == 0) - f0 = 0; - else - f0 = (float) Math.exp(logf0); - return f0; - } - } - - public boolean hasFeatureVector() { - return featureVector != null; - } - - public static UserDataHandler targetFeatureCloner = new UserDataHandler() { - public void handle(short operation, String key, Object data, Node src, Node dest) { - if (operation == UserDataHandler.NODE_CLONED && key == "target") { - dest.setUserData(key, data, this); - System.err.println("yay"); - } else { - System.err.println("nay"); - } - } - }; - - /** - * Determine whether this target is a silence target - * - * @return true if the target represents silence, false otherwise - */ - public boolean isSilence() { - - if (isSilence == -1) { - // TODO: how do we know the silence symbol here? - String silenceSymbol = "_"; - if (name.startsWith(silenceSymbol)) { - isSilence = 1; // true - } else { - isSilence = 0; // false - } - } - return isSilence == 1; - } - - public Allophone getAllophone() { - if (maryxmlElement != null) { - AllophoneSet allophoneSet = null; - Element voiceElement = (Element) MaryDomUtils.getAncestor(maryxmlElement, MaryXML.VOICE); - if (voiceElement != null) { - Voice v = Voice.getVoice(voiceElement); - if (v != null) { - allophoneSet = v.getAllophoneSet(); - } - } - if (allophoneSet == null) { - try { - allophoneSet = MaryRuntimeUtils.determineAllophoneSet(maryxmlElement); - } catch (Exception e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - } - String sampa; - if (maryxmlElement.getNodeName().equals(MaryXML.PHONE)) { - sampa = maryxmlElement.getAttribute("p"); - } else { - assert maryxmlElement.getNodeName().equals(MaryXML.BOUNDARY); - sampa = "_"; - } - return allophoneSet.getAllophone(sampa); - } - return null; - } - - public String toString() { - return name; - } -} diff --git a/marytts-unitselection/src/main/java/marytts/features/TargetFeatureComputer.java b/marytts-unitselection/src/main/java/marytts/features/TargetFeatureComputer.java deleted file mode 100644 index 078ab325e5..0000000000 --- a/marytts-unitselection/src/main/java/marytts/features/TargetFeatureComputer.java +++ /dev/null @@ -1,348 +0,0 @@ -/** - * Copyright 2008 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.features; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.StringReader; -import java.util.ArrayList; -import java.util.List; -import java.util.StringTokenizer; - - -/** - * Compute a given set of features for a Target. - * - * @author schroed - * - */ -public class TargetFeatureComputer { - protected ByteValuedFeatureProcessor[] byteValuedDiscreteFeatureProcessors; - protected ShortValuedFeatureProcessor[] shortValuedDiscreteFeatureProcessors; - protected ContinuousFeatureProcessor[] continuousFeatureProcessors; - - protected String pauseSymbol = null; - - protected FeatureDefinition featureDefinition = null; - - /** - * Construct a TargetFeatureComputer that knows how to compute features for a Target using the given set of feature processor - * names. These names must be known to the given Feature processor manager. - * - * @param manager - * @param featureProcessorNames - * a String containing the names of the feature processors to use, separated by white space, and in the right order - * (byte-valued discrete feature processors first, then short-valued, then continuous) - */ - public TargetFeatureComputer(FeatureProcessorManager manager, String featureProcessorNames) { - List byteValuedFeatureProcessors = new ArrayList(); - List shortValuedFeatureProcessors = new ArrayList(); - List continuousValuedFeatureProcessors = new ArrayList(); - - StringTokenizer st = new StringTokenizer(featureProcessorNames); - while (st.hasMoreTokens()) { - String name = st.nextToken(); - MaryFeatureProcessor fp = manager.getFeatureProcessor(name); - if (fp == null) { - throw new IllegalArgumentException("Unknown feature processor: " + name); - } else if (fp instanceof ByteValuedFeatureProcessor) { - byteValuedFeatureProcessors.add(fp); - } else if (fp instanceof ShortValuedFeatureProcessor) { - shortValuedFeatureProcessors.add(fp); - } else if (fp instanceof ContinuousFeatureProcessor) { - continuousValuedFeatureProcessors.add(fp); - } else { - throw new IllegalArgumentException("Unknown feature processor type " + fp.getClass() + " for feature processor: " - + name); - } - } - this.byteValuedDiscreteFeatureProcessors = (ByteValuedFeatureProcessor[]) byteValuedFeatureProcessors - .toArray(new ByteValuedFeatureProcessor[0]); - this.shortValuedDiscreteFeatureProcessors = (ShortValuedFeatureProcessor[]) shortValuedFeatureProcessors - .toArray(new ShortValuedFeatureProcessor[0]); - this.continuousFeatureProcessors = (ContinuousFeatureProcessor[]) continuousValuedFeatureProcessors - .toArray(new ContinuousFeatureProcessor[0]); - } - - /** - * Provide the feature definition that can be used to interpret the feature processors generated by this - * TargetFeatureComputer. - * - * @return - */ - public FeatureDefinition getFeatureDefinition() { - if (featureDefinition == null) { - StringBuilder sb = new StringBuilder(); - sb.append(FeatureDefinition.BYTEFEATURES).append("\n"); - for (int i = 0; i < byteValuedDiscreteFeatureProcessors.length; i++) { - sb.append(byteValuedDiscreteFeatureProcessors[i].getName()); - String[] values = byteValuedDiscreteFeatureProcessors[i].getValues(); - for (String v : values) { - sb.append(" ").append(v); - } - sb.append("\n"); - } - sb.append(FeatureDefinition.SHORTFEATURES).append("\n"); - for (int i = 0; i < shortValuedDiscreteFeatureProcessors.length; i++) { - sb.append(shortValuedDiscreteFeatureProcessors[i].getName()); - String[] values = shortValuedDiscreteFeatureProcessors[i].getValues(); - for (String v : values) { - sb.append(" ").append(v); - } - sb.append("\n"); - } - sb.append(FeatureDefinition.CONTINUOUSFEATURES).append("\n"); - for (int i = 0; i < continuousFeatureProcessors.length; i++) { - sb.append(continuousFeatureProcessors[i].getName()).append("\n"); - } - BufferedReader reader = new BufferedReader(new StringReader(sb.toString())); - try { - featureDefinition = new FeatureDefinition(reader, false); - } catch (IOException e) { - throw new RuntimeException("Problem creating feature definition", e); - } - } - return featureDefinition; - } - - /** - * Using the set of feature processors defined when creating the target feature computer, compute a feature vector for the - * target - * - * @param target - * @return a feature vector for the target - */ - public FeatureVector computeFeatureVector(Target target) { - byte[] byteFeatures = new byte[byteValuedDiscreteFeatureProcessors.length]; - short[] shortFeatures = new short[shortValuedDiscreteFeatureProcessors.length]; - float[] floatFeatures = new float[continuousFeatureProcessors.length]; - for (int i = 0; i < byteValuedDiscreteFeatureProcessors.length; i++) { - byteFeatures[i] = byteValuedDiscreteFeatureProcessors[i].process(target); - } - for (int i = 0; i < shortValuedDiscreteFeatureProcessors.length; i++) { - shortFeatures[i] = shortValuedDiscreteFeatureProcessors[i].process(target); - } - for (int i = 0; i < continuousFeatureProcessors.length; i++) { - floatFeatures[i] = continuousFeatureProcessors[i].process(target); - } - return new FeatureVector(byteFeatures, shortFeatures, floatFeatures, 0); - } - - /** - * For the given feature vector, convert each encoded value into its string representation. - * - * @param features - * a feature vector, which must match the feature processors known to this feature computer. - * @return a string in which the string values of all features are separated by spaces. - * @throws IllegalArgumentException - * if the number of byte-valued, short-valued or continuous elements in features do not match the set of feature - * processors in this feature computer. - */ - public String toStringValues(FeatureVector features) { - StringBuilder buf = new StringBuilder(); - byte[] bytes = features.getByteValuedDiscreteFeatures(); - short[] shorts = features.getShortValuedDiscreteFeatures(); - float[] floats = features.getContinuousFeatures(); - if (bytes.length != byteValuedDiscreteFeatureProcessors.length - || shorts.length != shortValuedDiscreteFeatureProcessors.length - || floats.length != continuousFeatureProcessors.length) { - throw new IllegalArgumentException("Number of features in argument does not match number of feature processors"); - } - for (int i = 0; i < bytes.length; i++) { - if (buf.length() > 0) - buf.append(" "); - buf.append(byteValuedDiscreteFeatureProcessors[i].getValues()[(int) bytes[i] & 0xff]); - } - for (int i = 0; i < shorts.length; i++) { - if (buf.length() > 0) - buf.append(" "); - buf.append(shortValuedDiscreteFeatureProcessors[i].getValues()[(int) shorts[i]]); - } - for (int i = 0; i < floats.length; i++) { - if (buf.length() > 0) - buf.append(" "); - buf.append(floats[i]); - } - return buf.toString(); - } - - public ByteValuedFeatureProcessor[] getByteValuedFeatureProcessors() { - return byteValuedDiscreteFeatureProcessors; - } - - public ShortValuedFeatureProcessor[] getShortValuedFeatureProcessors() { - return shortValuedDiscreteFeatureProcessors; - } - - public ContinuousFeatureProcessor[] getContinuousFeatureProcessors() { - return continuousFeatureProcessors; - } - - /** - * List the names of all feature processors. The first line starts with "ByteValuedFeatureProcessors", followed by the list of - * names of the byte-valued feature processors; the second line starts with "ShortValuedFeatureProcessors", followed by the - * list of names of the short-valued feature processors; and the third line starts with "ContinuousFeatureProcessors", - * followed by the list of names of the continuous feature processors. - * - * @return a string with the names. - */ - public String getAllFeatureProcessorNames() { - return "ByteValuedFeatureProcessors " + getByteValuedFeatureProcessorNames() + "\n" + "ShortValuedFeatureProcessors " - + getShortValuedFeatureProcessorNames() + "\n" + "ContinuousFeatureProcessors " - + getContinuousFeatureProcessorNames() + "\n"; - } - - /** - * List the names of all byte-valued feature processors, separated by space characters. - * - * @return a string with the names. - */ - public String getByteValuedFeatureProcessorNames() { - StringBuilder buf = new StringBuilder(); - for (int i = 0; i < byteValuedDiscreteFeatureProcessors.length; i++) { - if (i > 0) - buf.append(" "); - buf.append(byteValuedDiscreteFeatureProcessors[i].getName()); - } - return buf.toString(); - } - - /** - * List the names of all short-valued feature processors, separated by space characters. - * - * @return a string with the names. - */ - public String getShortValuedFeatureProcessorNames() { - StringBuilder buf = new StringBuilder(); - for (int i = 0; i < shortValuedDiscreteFeatureProcessors.length; i++) { - if (i > 0) - buf.append(" "); - buf.append(shortValuedDiscreteFeatureProcessors[i].getName()); - } - return buf.toString(); - } - - /** - * List the names of all byte-valued feature processors, separated by space characters. - * - * @return a string with the names. - */ - public String getContinuousFeatureProcessorNames() { - StringBuilder buf = new StringBuilder(); - for (int i = 0; i < continuousFeatureProcessors.length; i++) { - if (i > 0) - buf.append(" "); - buf.append(continuousFeatureProcessors[i].getName()); - } - return buf.toString(); - } - - /** - * List the names and values of all feature processors. The section describing the byte-valued feature processors starts with - * the string "ByteValuedFeatureProcessors" in a line by itself, followed by the list of names and values of the byte-valued - * feature processors, as described in getByteValuedFeatureProcessorNamesAndValues(). The section describing the short-valued - * feature processors starts with the string "ShortValuedFeatureProcessors" in a line by itself, followed by the list of names - * and values of the short-valued feature processors, as described in getShortValuedFeatureProcessorNamesAndValues(). The - * section describing the continuous feature processors starts with the string "ContinuousFeatureProcessors" in a line by - * itself, followed by the list of names and values of the continuous feature processors, as described in - * getContinuousFeatureProcessorNamesAndValues(). - * - * @return a string with the names and values. - */ - public String getAllFeatureProcessorNamesAndValues() { - return FeatureDefinition.BYTEFEATURES + "\n" + getByteValuedFeatureProcessorNamesAndValues() - + FeatureDefinition.SHORTFEATURES + "\n" + getShortValuedFeatureProcessorNamesAndValues() - + FeatureDefinition.CONTINUOUSFEATURES + "\n" + getContinuousFeatureProcessorNamesAndValues(); - } - - /** - * List the names of all byte-valued feature processors and their possible values. Each line starts with the name of a feature - * processor, followed by the full list of string values, separated by space characters. The values are ordered so that their - * position corresponds to the byte value. - * - * @return a string with the names and values. - */ - public String getByteValuedFeatureProcessorNamesAndValues() { - StringBuilder buf = new StringBuilder(); - for (int i = 0; i < byteValuedDiscreteFeatureProcessors.length; i++) { - buf.append(byteValuedDiscreteFeatureProcessors[i].getName()); - String[] values = byteValuedDiscreteFeatureProcessors[i].getValues(); - for (int j = 0; j < values.length; j++) { - buf.append(" "); - buf.append(values[j]); - } - buf.append("\n"); - } - return buf.toString(); - } - - /** - * List the names of all short-valued feature processors and their possible values. Each line starts with the name of a - * feature processor, followed by the full list of string values, separated by space characters. The values are ordered so - * that their position corresponds to the short value. - * - * @return a string with the names and values. - */ - public String getShortValuedFeatureProcessorNamesAndValues() { - StringBuilder buf = new StringBuilder(); - for (int i = 0; i < shortValuedDiscreteFeatureProcessors.length; i++) { - buf.append(shortValuedDiscreteFeatureProcessors[i].getName()); - String[] values = shortValuedDiscreteFeatureProcessors[i].getValues(); - for (int j = 0; j < values.length; j++) { - buf.append(" "); - buf.append(values[j]); - } - buf.append("\n"); - } - return buf.toString(); - } - - /** - * List the names of all continuous feature processors and their possible values. Each line starts with the name of a feature - * processor, followed by the value "float" to indicate that the list of possible values is not limited. - * - * @return a string with the names and values. - */ - public String getContinuousFeatureProcessorNamesAndValues() { - StringBuilder buf = new StringBuilder(); - for (int i = 0; i < continuousFeatureProcessors.length; i++) { - buf.append(continuousFeatureProcessors[i].getName()); - buf.append(" float\n"); - } - return buf.toString(); - } - - /** - * Get the pause symbol as associated with the "phone" feature processor used. - * - * @return - */ - public String getPauseSymbol() { - if (pauseSymbol == null) { - for (MaryFeatureProcessor fp : byteValuedDiscreteFeatureProcessors) { - if (fp instanceof MaryLanguageFeatureProcessors.Phone) { - pauseSymbol = ((MaryLanguageFeatureProcessors.Phone) fp).getPauseSymbol(); - break; - } - } - } - return pauseSymbol; - } -} From 8841eff00cacb2e13c8326e6e968c7c3051a3442 Mon Sep 17 00:00:00 2001 From: Tristan Hamilton Date: Fri, 6 Feb 2015 14:15:38 +0100 Subject: [PATCH 20/31] remove duplicate cart packages from unitselection module --- .../src/main/java/marytts/cart/CART.java | 177 ----- .../main/java/marytts/cart/DecisionNode.java | 748 ------------------ .../main/java/marytts/cart/DirectedGraph.java | 276 ------- .../java/marytts/cart/DirectedGraphNode.java | 218 ----- .../java/marytts/cart/FeatureVectorCART.java | 129 --- .../src/main/java/marytts/cart/LeafNode.java | 541 ------------- .../src/main/java/marytts/cart/Node.java | 172 ---- .../main/java/marytts/cart/NodeIterator.java | 163 ---- .../marytts/cart/StringPredictionTree.java | 222 ------ .../cart/impose/FeatureArrayIndexer.java | 422 ---------- .../cart/impose/FeatureComparator.java | 118 --- .../impose/FeatureFileIndexingResult.java | 46 -- .../java/marytts/cart/impose/MaryNode.java | 133 ---- .../marytts/cart/io/DirectedGraphReader.java | 290 ------- .../marytts/cart/io/DirectedGraphWriter.java | 409 ---------- .../java/marytts/cart/io/HTSCARTReader.java | 556 ------------- .../java/marytts/cart/io/MaryCARTReader.java | 402 ---------- .../java/marytts/cart/io/MaryCARTWriter.java | 345 -------- .../java/marytts/cart/io/WagonCARTReader.java | 607 -------------- .../java/marytts/cart/io/WagonCARTWriter.java | 379 --------- 20 files changed, 6353 deletions(-) delete mode 100644 marytts-unitselection/src/main/java/marytts/cart/CART.java delete mode 100644 marytts-unitselection/src/main/java/marytts/cart/DecisionNode.java delete mode 100644 marytts-unitselection/src/main/java/marytts/cart/DirectedGraph.java delete mode 100644 marytts-unitselection/src/main/java/marytts/cart/DirectedGraphNode.java delete mode 100644 marytts-unitselection/src/main/java/marytts/cart/FeatureVectorCART.java delete mode 100644 marytts-unitselection/src/main/java/marytts/cart/LeafNode.java delete mode 100644 marytts-unitselection/src/main/java/marytts/cart/Node.java delete mode 100644 marytts-unitselection/src/main/java/marytts/cart/NodeIterator.java delete mode 100644 marytts-unitselection/src/main/java/marytts/cart/StringPredictionTree.java delete mode 100644 marytts-unitselection/src/main/java/marytts/cart/impose/FeatureArrayIndexer.java delete mode 100644 marytts-unitselection/src/main/java/marytts/cart/impose/FeatureComparator.java delete mode 100644 marytts-unitselection/src/main/java/marytts/cart/impose/FeatureFileIndexingResult.java delete mode 100644 marytts-unitselection/src/main/java/marytts/cart/impose/MaryNode.java delete mode 100644 marytts-unitselection/src/main/java/marytts/cart/io/DirectedGraphReader.java delete mode 100644 marytts-unitselection/src/main/java/marytts/cart/io/DirectedGraphWriter.java delete mode 100644 marytts-unitselection/src/main/java/marytts/cart/io/HTSCARTReader.java delete mode 100644 marytts-unitselection/src/main/java/marytts/cart/io/MaryCARTReader.java delete mode 100644 marytts-unitselection/src/main/java/marytts/cart/io/MaryCARTWriter.java delete mode 100644 marytts-unitselection/src/main/java/marytts/cart/io/WagonCARTReader.java delete mode 100644 marytts-unitselection/src/main/java/marytts/cart/io/WagonCARTWriter.java diff --git a/marytts-unitselection/src/main/java/marytts/cart/CART.java b/marytts-unitselection/src/main/java/marytts/cart/CART.java deleted file mode 100644 index ac65c1ad85..0000000000 --- a/marytts-unitselection/src/main/java/marytts/cart/CART.java +++ /dev/null @@ -1,177 +0,0 @@ -/** - * Portions Copyright 2006 DFKI GmbH. - * Portions Copyright 2001 Sun Microsystems, Inc. - * Portions Copyright 1999-2001 Language Technologies Institute, - * Carnegie Mellon University. - * All Rights Reserved. Use is subject to license terms. - * - * Permission is hereby granted, free of charge, to use and distribute - * this software and its documentation without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of this work, and to - * permit persons to whom this work is furnished to do so, subject to - * the following conditions: - * - * 1. The code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * 2. Any modifications must be clearly marked as such. - * 3. Original authors' names are not deleted. - * 4. The authors' names are not used to endorse or promote products - * derived from this software without specific prior written - * permission. - * - * DFKI GMBH AND THE CONTRIBUTORS TO THIS WORK DISCLAIM ALL WARRANTIES WITH - * REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL DFKI GMBH NOR THE - * CONTRIBUTORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL - * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR - * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS - * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF - * THIS SOFTWARE. - */ -package marytts.cart; - -import java.util.Properties; - -import marytts.features.FeatureDefinition; -import marytts.features.FeatureVector; -import marytts.features.Target; - -/** - * A tree is a specific kind of directed graph in which each node can have only a single parent node. It consists exclusively of - * DecisionNode and LeafNode nodes. - * - * @author marc - * - */ -public class CART extends DirectedGraph { - - /** - * Build a new empty cart - * - */ - public CART() { - } - - /** - * Build a new empty cart with the given feature definition. - * - * @param featDef - */ - public CART(FeatureDefinition featDef) { - super(featDef); - } - - /** - * Build a new cart with the given node as the root node - * - * @param rootNode - * the root node of the CART - * @param featDef - * the feature definition used for interpreting the meaning of decision node criteria. - */ - public CART(Node rootNode, FeatureDefinition featDef) { - super(rootNode, featDef); - } - - /** - * Build a new cart with the given node as the root node - * - * @param rootNode - * the root node of the CART - * @param featDef - * the feature definition used for interpreting the meaning of decision node criteria. - * @param properties - * a generic properties object, which can be used to encode information about the tree and the way the data in it - * should be represented. - */ - public CART(Node rootNode, FeatureDefinition featDef, Properties properties) { - super(rootNode, featDef, properties); - } - - /** - * Passes the given item through this CART and returns the leaf Node, or the Node it stopped walking down. - * - * @param target - * the target to analyze - * @param minNumberOfData - * the minimum number of data requested. If this is 0, walk down the CART until the leaf level. - * - * @return the Node - */ - public Node interpretToNode(Target target, int minNumberOfData) { - return interpretToNode(target.getFeatureVector(), minNumberOfData); - } - - /** - * Passes the given item through this CART and returns the leaf Node, or the Node it stopped walking down. - * - * @param target - * the target to analyze - * @param minNumberOfData - * the minimum number of data requested. If this is 0, walk down the CART until the leaf level. - * - * @return the Node - */ - public Node interpretToNode(FeatureVector featureVector, int minNumberOfData) { - Node currentNode = rootNode; - Node prevNode = null; - - // logger.debug("Starting cart at "+nodeIndex); - while (currentNode != null && currentNode.getNumberOfData() > minNumberOfData && !(currentNode instanceof LeafNode)) { - // while we have not reached the bottom, - // get the next node based on the features of the target - prevNode = currentNode; - currentNode = ((DecisionNode) currentNode).getNextNode(featureVector); - // logger.debug(decision.toString() + " result '"+ - // decision.findFeature(item) + "' => "+ nodeIndex); - } - // Now usually we will have gone down one level too far - if (currentNode == null || currentNode.getNumberOfData() < minNumberOfData && prevNode != null) { - currentNode = prevNode; - } - - assert currentNode.getNumberOfData() >= minNumberOfData || currentNode == rootNode; - - assert minNumberOfData > 0 || (currentNode instanceof LeafNode); - return currentNode; - - } - - /** - * Passes the given item through this CART and returns the interpretation. - * - * @param target - * the target to analyze - * @param minNumberOfData - * the minimum number of data requested. If this is 0, walk down the CART until the leaf level. - * - * @return the interpretation - */ - public Object interpret(Target target, int minNumberOfData) { - - // get the indices from the leaf node - Object result = this.interpretToNode(target, minNumberOfData).getAllData(); - - return result; - - } - - /** - * In this tree, replace the given leaf with the given CART - * - * @param cart - * the CART - * @param leaf - * the leaf - * @return the ex-root node from cart which now replaces leaf. - */ - public static Node replaceLeafByCart(CART cart, LeafNode leaf) { - DecisionNode mother = (DecisionNode) leaf.getMother(); - Node newNode = cart.getRootNode(); - mother.replaceDaughter(newNode, leaf.getNodeIndex()); - newNode.setIsRoot(false); - return newNode; - } - -} diff --git a/marytts-unitselection/src/main/java/marytts/cart/DecisionNode.java b/marytts-unitselection/src/main/java/marytts/cart/DecisionNode.java deleted file mode 100644 index 8cb0b8506c..0000000000 --- a/marytts-unitselection/src/main/java/marytts/cart/DecisionNode.java +++ /dev/null @@ -1,748 +0,0 @@ -/** - * Copyright 2000-2009 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.cart; - -import marytts.cart.LeafNode.FeatureVectorLeafNode; -import marytts.cart.LeafNode.IntArrayLeafNode; -import marytts.features.FeatureDefinition; -import marytts.features.FeatureVector; - -/** - * A decision node that determines the next Node to go to in the CART. All decision nodes inherit from this class - */ -public abstract class DecisionNode extends Node { - public enum Type { - BinaryByteDecisionNode, BinaryShortDecisionNode, BinaryFloatDecisionNode, ByteDecisionNode, ShortDecisionNode - }; - - protected boolean TRACE = false; - // for debugging: - protected FeatureDefinition featureDefinition; - - // a decision node has an array of daughters - protected Node[] daughters; - - // the feature index - protected int featureIndex; - - // the feature name - protected String feature; - - // remember last added daughter - protected int lastDaughter; - - // the total number of data in the leaves below this node - protected int nData; - - // unique index used in MaryCART format - protected int uniqueDecisionNodeId; - - /** - * Construct a new DecisionNode - * - * @param feature - * the feature - * @param numDaughters - * the number of daughters - */ - public DecisionNode(String feature, int numDaughters, FeatureDefinition featureDefinition) { - this.feature = feature; - this.featureIndex = featureDefinition.getFeatureIndex(feature); - daughters = new Node[numDaughters]; - isRoot = false; - // for trace and getDecisionPath(): - this.featureDefinition = featureDefinition; - } - - /** - * Construct a new DecisionNode - * - * @param featureIndex - * the feature index - * @param numDaughters - * the number of daughters - */ - public DecisionNode(int featureIndex, int numDaughters, FeatureDefinition featureDefinition) { - this.featureIndex = featureIndex; - this.feature = featureDefinition.getFeatureName(featureIndex); - daughters = new Node[numDaughters]; - isRoot = false; - // for trace and getDecisionPath(): - this.featureDefinition = featureDefinition; - } - - /** - * Construct a new DecisionNode - * - * @param numDaughters - * the number of daughters - */ - public DecisionNode(int numDaughters, FeatureDefinition featureDefinition) { - daughters = new Node[numDaughters]; - isRoot = false; - // for trace and getDecisionPath(): - this.featureDefinition = featureDefinition; - } - - @Override - public boolean isDecisionNode() { - return true; - } - - /** - * Get the name of the feature - * - * @return the name of the feature - */ - public String getFeatureName() { - return feature; - } - - public int getFeatureIndex() { - return featureIndex; - } - - public FeatureDefinition getFeatureDefinition() { - return featureDefinition; - } - - /** - * Add a daughter to the node - * - * @param daughter - * the new daughter - */ - public void addDaughter(Node daughter) { - if (lastDaughter > daughters.length - 1) { - throw new RuntimeException("Can not add daughter number " + (lastDaughter + 1) + ", since node has only " - + daughters.length + " daughters!"); - } - daughters[lastDaughter] = daughter; - if (daughter != null) { - daughter.setMother(this, lastDaughter); - } - lastDaughter++; - } - - /** - * Get the daughter at the specified index - * - * @param index - * the index of the daughter - * @return the daughter (potentially null); if index out of range: null - */ - public Node getDaughter(int index) { - if (index > daughters.length - 1 || index < 0) { - return null; - } - return daughters[index]; - } - - /** - * Replace daughter at given index with another daughter - * - * @param newDaughter - * the new daughter - * @param index - * the index of the daughter to replace - */ - public void replaceDaughter(Node newDaughter, int index) { - if (index > daughters.length - 1 || index < 0) { - throw new RuntimeException("Can not replace daughter number " + index + ", since daughter index goes from 0 to " - + (daughters.length - 1) + "!"); - } - daughters[index] = newDaughter; - newDaughter.setMother(this, index); - } - - /** - * Tests, if the given index refers to a daughter - * - * @param index - * the index - * @return true, if the index is in range of the daughters array - */ - public boolean hasMoreDaughters(int index) { - return (index > -1 && index < daughters.length); - } - - /** - * Get all unit indices from all leaves below this node - * - * @return an int array containing the indices - */ - public Object getAllData() { - // What to do depends on the type of leaves. - LeafNode firstLeaf = new NodeIterator(this, true, false, false).next(); - if (firstLeaf == null) - return null; - Object result; - if (firstLeaf instanceof IntArrayLeafNode) { // this includes subclass IntAndFloatArrayLeafNode - result = new int[nData]; - } else if (firstLeaf instanceof FeatureVectorLeafNode) { - result = new FeatureVector[nData]; - } else { - return null; - } - fillData(result, 0, nData); - return result; - } - - protected void fillData(Object target, int pos, int total) { - // assert pos + total <= target.length; - for (int i = 0; i < daughters.length; i++) { - if (daughters[i] == null) - continue; - int len = daughters[i].getNumberOfData(); - daughters[i].fillData(target, pos, len); - pos += len; - } - } - - /** - * Count all the nodes at and below this node. A leaf will return 1; the root node will report the total number of decision - * and leaf nodes in the tree. - * - * @return - */ - public int getNumberOfNodes() { - int nNodes = 1; // this node - for (int i = 0; i < daughters.length; i++) { - if (daughters[i] != null) - nNodes += daughters[i].getNumberOfNodes(); - } - return nNodes; - } - - public int getNumberOfData() { - return nData; - } - - /** Number of daughters of current node. */ - public int getNumberOfDaugthers() { - return daughters.length; - } - - /** - * Set the number of candidates correctly, by counting while walking down the tree. This needs to be done once for the entire - * tree. - * - */ - // protected void countData() { - public void countData() { - nData = 0; - for (int i = 0; i < daughters.length; i++) { - if (daughters[i] instanceof DecisionNode) - ((DecisionNode) daughters[i]).countData(); - if (daughters[i] != null) { - nData += daughters[i].getNumberOfData(); - } - } - } - - public String toString() { - return "dn" + uniqueDecisionNodeId; - } - - /** - * Get the path leading to the daughter with the given index. This will recursively go up to the root node. - * - * @param daughterIndex - * @return - */ - public abstract String getDecisionPath(int daughterIndex); - - // unique index used in MaryCART format - public void setUniqueDecisionNodeId(int id) { - this.uniqueDecisionNodeId = id; - } - - public int getUniqueDecisionNodeId() { - return uniqueDecisionNodeId; - } - - /** - * Gets the String that defines the decision done in the node - * - * @return the node definition - */ - public abstract String getNodeDefinition(); - - /** - * Get the decision node type - * - * @return - */ - public abstract Type getDecisionNodeType(); - - /** - * Select a daughter node according to the value in the given target - * - * @param target - * the target - * @return a daughter - */ - public abstract Node getNextNode(FeatureVector featureVector); - - /** - * A binary decision Node that compares two byte values. - */ - public static class BinaryByteDecisionNode extends DecisionNode { - - // the value of this node - private byte value; - - /** - * Create a new binary String DecisionNode. - * - * @param feature - * the string used to get a value from an Item - * @param value - * the value to compare to - */ - public BinaryByteDecisionNode(String feature, String value, FeatureDefinition featureDefinition) { - super(feature, 2, featureDefinition); - this.value = featureDefinition.getFeatureValueAsByte(feature, value); - } - - public BinaryByteDecisionNode(int featureIndex, byte value, FeatureDefinition featureDefinition) { - super(featureIndex, 2, featureDefinition); - this.value = value; - } - - /*** - * Creates an empty BinaryByteDecisionNode, the feature and feature value of this node should be filled with - * setFeatureAndFeatureValue() function. - * - * @param uniqueId - * unique index from tree HTS test file. - * @param featureDefinition - */ - public BinaryByteDecisionNode(int uniqueId, FeatureDefinition featureDefinition) { - super(2, featureDefinition); - // System.out.println("adding decision node: " + uniqueId); - this.uniqueDecisionNodeId = uniqueId; - } - - /*** - * Fill the feature and feature value of an already created (empty) BinaryByteDecisionNode. - * - * @param feature - * @param value - */ - public void setFeatureAndFeatureValue(String feature, String value) { - this.feature = feature; - this.featureIndex = featureDefinition.getFeatureIndex(feature); - this.value = featureDefinition.getFeatureValueAsByte(feature, value); - } - - public byte getCriterionValueAsByte() { - return value; - } - - public String getCriterionValueAsString() { - return featureDefinition.getFeatureValueAsString(featureIndex, value); - } - - /** - * Select a daughter node according to the value in the given target - * - * @param target - * the target - * @return a daughter - */ - public Node getNextNode(FeatureVector featureVector) { - byte val = featureVector.getByteFeature(featureIndex); - Node returnNode; - if (val == value) { - returnNode = daughters[0]; - } else { - returnNode = daughters[1]; - } - if (TRACE) { - System.out.print(" " + feature + ": " + featureDefinition.getFeatureValueAsString(featureIndex, value) - + " == " + featureDefinition.getFeatureValueAsString(featureIndex, val)); - if (val == value) - System.out.println(" YES "); - else - System.out.println(" NO "); - } - return returnNode; - } - - public String getDecisionPath(int daughterIndex) { - String thisNodeInfo; - if (daughterIndex == 0) - thisNodeInfo = feature + "==" + featureDefinition.getFeatureValueAsString(featureIndex, value); - else - thisNodeInfo = feature + "!=" + featureDefinition.getFeatureValueAsString(featureIndex, value); - if (mother == null) - return thisNodeInfo; - else if (mother.isDecisionNode()) - return ((DecisionNode) mother).getDecisionPath(getNodeIndex()) + " - " + thisNodeInfo; - else - return mother.getDecisionPath() + " - " + thisNodeInfo; - } - - /** - * Gets the String that defines the decision done in the node - * - * @return the node definition - */ - public String getNodeDefinition() { - return feature + " is " + featureDefinition.getFeatureValueAsString(featureIndex, value); - } - - public Type getDecisionNodeType() { - return Type.BinaryByteDecisionNode; - } - - } - - /** - * A binary decision Node that compares two short values. - */ - public static class BinaryShortDecisionNode extends DecisionNode { - - // the value of this node - private short value; - - /** - * Create a new binary String DecisionNode. - * - * @param feature - * the string used to get a value from an Item - * @param value - * the value to compare to - */ - public BinaryShortDecisionNode(String feature, String value, FeatureDefinition featureDefinition) { - super(feature, 2, featureDefinition); - this.value = featureDefinition.getFeatureValueAsShort(feature, value); - } - - public BinaryShortDecisionNode(int featureIndex, short value, FeatureDefinition featureDefinition) { - super(featureIndex, 2, featureDefinition); - this.value = value; - } - - public short getCriterionValueAsShort() { - return value; - } - - public String getCriterionValueAsString() { - return featureDefinition.getFeatureValueAsString(featureIndex, value); - } - - /** - * Select a daughter node according to the value in the given target - * - * @param target - * the target - * @return a daughter - */ - public Node getNextNode(FeatureVector featureVector) { - short val = featureVector.getShortFeature(featureIndex); - Node returnNode; - if (val == value) { - returnNode = daughters[0]; - } else { - returnNode = daughters[1]; - } - if (TRACE) { - System.out.print(feature + ": " + featureDefinition.getFeatureValueAsString(featureIndex, val)); - if (val == value) - System.out.print(" == "); - else - System.out.print(" != "); - System.out.println(featureDefinition.getFeatureValueAsString(featureIndex, value)); - } - return returnNode; - } - - public String getDecisionPath(int daughterIndex) { - String thisNodeInfo; - if (daughterIndex == 0) - thisNodeInfo = feature + "==" + featureDefinition.getFeatureValueAsString(featureIndex, value); - else - thisNodeInfo = feature + "!=" + featureDefinition.getFeatureValueAsString(featureIndex, value); - if (mother == null) - return thisNodeInfo; - else if (mother.isDecisionNode()) - return ((DecisionNode) mother).getDecisionPath(getNodeIndex()) + " - " + thisNodeInfo; - else - return mother.getDecisionPath() + " - " + thisNodeInfo; - } - - /** - * Gets the String that defines the decision done in the node - * - * @return the node definition - */ - public String getNodeDefinition() { - return feature + " is " + featureDefinition.getFeatureValueAsString(featureIndex, value); - } - - public Type getDecisionNodeType() { - return Type.BinaryShortDecisionNode; - } - - } - - /** - * A binary decision Node that compares two float values. - */ - public static class BinaryFloatDecisionNode extends DecisionNode { - - // the value of this node - private float value; - private boolean isByteFeature; - - /** - * Create a new binary String DecisionNode. - * - * @param feature - * the string used to get a value from an Item - * @param value - * the value to compare to - */ - public BinaryFloatDecisionNode(int featureIndex, float value, FeatureDefinition featureDefinition) { - this(featureDefinition.getFeatureName(featureIndex), value, featureDefinition); - } - - public BinaryFloatDecisionNode(String feature, float value, FeatureDefinition featureDefinition) { - super(feature, 2, featureDefinition); - this.value = value; - // check for pseudo-floats: - // TODO: clean this up: - if (featureDefinition.isByteFeature(featureIndex)) - isByteFeature = true; - else - isByteFeature = false; - } - - public float getCriterionValueAsFloat() { - return value; - } - - public String getCriterionValueAsString() { - return String.valueOf(value); - } - - /** - * Select a daughter node according to the value in the given target - * - * @param target - * the target - * @return a daughter - */ - public Node getNextNode(FeatureVector featureVector) { - float val; - if (isByteFeature) - val = (float) featureVector.getByteFeature(featureIndex); - else - val = featureVector.getContinuousFeature(featureIndex); - Node returnNode; - if (val < value) { - returnNode = daughters[0]; - } else { - returnNode = daughters[1]; - } - if (TRACE) { - System.out.print(feature + ": " + val); - if (val < value) - System.out.print(" < "); - else - System.out.print(" >= "); - System.out.println(value); - } - - return returnNode; - } - - public String getDecisionPath(int daughterIndex) { - String thisNodeInfo; - if (daughterIndex == 0) - thisNodeInfo = feature + "<" + value; - else - thisNodeInfo = feature + ">=" + value; - if (mother == null) - return thisNodeInfo; - else if (mother.isDecisionNode()) - return ((DecisionNode) mother).getDecisionPath(getNodeIndex()) + " - " + thisNodeInfo; - else - return mother.getDecisionPath() + " - " + thisNodeInfo; - } - - /** - * Gets the String that defines the decision done in the node - * - * @return the node definition - */ - public String getNodeDefinition() { - return feature + " < " + value; - } - - public Type getDecisionNodeType() { - return Type.BinaryFloatDecisionNode; - } - - } - - /** - * An decision Node with an arbitrary number of daughters. Value of the target corresponds to the index number of next - * daughter. - */ - public static class ByteDecisionNode extends DecisionNode { - - /** - * Build a new byte decision node - * - * @param feature - * the feature name - * @param numDaughters - * the number of daughters - */ - public ByteDecisionNode(String feature, int numDaughters, FeatureDefinition featureDefinition) { - super(feature, numDaughters, featureDefinition); - } - - /** - * Build a new byte decision node - * - * @param feature - * the feature name - * @param numDaughters - * the number of daughters - */ - public ByteDecisionNode(int featureIndex, int numDaughters, FeatureDefinition featureDefinition) { - super(featureIndex, numDaughters, featureDefinition); - } - - /** - * Select a daughter node according to the value in the given target - * - * @param target - * the target - * @return a daughter - */ - public Node getNextNode(FeatureVector featureVector) { - byte val = featureVector.getByteFeature(featureIndex); - if (TRACE) { - System.out.println(feature + ": " + featureDefinition.getFeatureValueAsString(featureIndex, val)); - } - return daughters[val]; - } - - public String getDecisionPath(int daughterIndex) { - String thisNodeInfo = feature + "==" + featureDefinition.getFeatureValueAsString(featureIndex, daughterIndex); - if (mother == null) - return thisNodeInfo; - else if (mother.isDecisionNode()) - return ((DecisionNode) mother).getDecisionPath(getNodeIndex()) + " - " + thisNodeInfo; - else - return mother.getDecisionPath() + " - " + thisNodeInfo; - } - - /** - * Gets the String that defines the decision done in the node - * - * @return the node definition - */ - public String getNodeDefinition() { - return feature + " isByteOf " + daughters.length; - } - - public Type getDecisionNodeType() { - return Type.ByteDecisionNode; - } - - } - - /** - * An decision Node with an arbitrary number of daughters. Value of the target corresponds to the index number of next - * daughter. - */ - public static class ShortDecisionNode extends DecisionNode { - - /** - * Build a new short decision node - * - * @param feature - * the feature name - * @param numDaughters - * the number of daughters - */ - public ShortDecisionNode(String feature, int numDaughters, FeatureDefinition featureDefinition) { - super(feature, numDaughters, featureDefinition); - } - - /** - * Build a new short decision node - * - * @param featureIndex - * the feature index - * @param numDaughters - * the number of daughters - */ - public ShortDecisionNode(int featureIndex, int numDaughters, FeatureDefinition featureDefinition) { - super(featureIndex, numDaughters, featureDefinition); - } - - /** - * Select a daughter node according to the value in the given target - * - * @param target - * the target - * @return a daughter - */ - public Node getNextNode(FeatureVector featureVector) { - short val = featureVector.getShortFeature(featureIndex); - if (TRACE) { - System.out.println(feature + ": " + featureDefinition.getFeatureValueAsString(featureIndex, val)); - } - return daughters[val]; - } - - public String getDecisionPath(int daughterIndex) { - String thisNodeInfo = feature + "==" + featureDefinition.getFeatureValueAsString(featureIndex, daughterIndex); - if (mother == null) - return thisNodeInfo; - else if (mother.isDecisionNode()) - return ((DecisionNode) mother).getDecisionPath(getNodeIndex()) + " - " + thisNodeInfo; - else - return mother.getDecisionPath() + " - " + thisNodeInfo; - } - - /** - * Gets the String that defines the decision done in the node - * - * @return the node definition - */ - public String getNodeDefinition() { - return feature + " isShortOf " + daughters.length; - } - - public Type getDecisionNodeType() { - return Type.ShortDecisionNode; - } - - } - -} diff --git a/marytts-unitselection/src/main/java/marytts/cart/DirectedGraph.java b/marytts-unitselection/src/main/java/marytts/cart/DirectedGraph.java deleted file mode 100644 index 3d3e1db64e..0000000000 --- a/marytts-unitselection/src/main/java/marytts/cart/DirectedGraph.java +++ /dev/null @@ -1,276 +0,0 @@ -/** - * Copyright 2009 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ - -package marytts.cart; - -import java.util.Iterator; -import java.util.Properties; - -import marytts.features.FeatureDefinition; -import marytts.features.FeatureVector; -import marytts.features.Target; -import marytts.util.MaryUtils; - -import org.apache.log4j.Logger; - -/** - * A directed graph is a layered structure of nodes, in which there are mother-daughter relationships between the node. There is a - * single root node. Each node can have multiple daughters and/or multiple mothers. Three types of nodes are allowed: - * DirectedGraphNode (which can have multiple mothers, a leaf and a decision node), LeafNodes (which carry data), and - * DecisionNodes (which can have multiple daughters). - * - * @author marc - * - */ -public class DirectedGraph { - protected Logger logger = MaryUtils.getLogger(this.getClass().getName()); - - protected Node rootNode; - - // knows the index numbers and types of the features used in DecisionNodes - protected FeatureDefinition featDef; - - protected Properties properties; - - /** - * Build a new empty directed graph - * - */ - public DirectedGraph() { - } - - /** - * Build a new empty graph with the given feature definition. - * - * @param featDef - * the feature definition used for interpreting the meaning of decision node criteria. - */ - public DirectedGraph(FeatureDefinition featDef) { - this(null, featDef); - } - - /** - * Build a new graph with the given node as the root node - * - * @param rootNode - * the root node of the graph - * @param featDef - * the feature definition used for interpreting the meaning of decision node criteria. - */ - public DirectedGraph(Node rootNode, FeatureDefinition featDef) { - this(rootNode, featDef, null); - } - - /** - * Build a new graph with the given node as the root node - * - * @param rootNode - * the root node of the graph - * @param featDef - * the feature definition used for interpreting the meaning of decision node criteria. - * @param properties - * a generic properties object, which can be used to encode information about the tree and the way the data in it - * should be represented. - */ - public DirectedGraph(Node rootNode, FeatureDefinition featDef, Properties properties) { - this.rootNode = rootNode; - this.featDef = featDef; - this.properties = properties; - } - - public Object interpret(Target t) { - return interpret(t.getFeatureVector()); - } - - /** - * Walk down the graph as far as possible according to the features in fv, and return the data in the leaf node found there. - * - * @param fv - * a feature vector which must be consistent with the graph's feature definition. (@see #getFeatureDefinition()). - * @return the most specific non-null leaf node data that can be retrieved, or null if there is no non-null leaf node data - * along the fv's path. - */ - public Object interpret(FeatureVector fv) { - return interpret(rootNode, fv); - } - - /** - * Follow the directed graph down to the most specific leaf with data, starting from node n. This is recursively calling - * itself. - * - * @param n - * @param fv - * @return - */ - protected Object interpret(Node n, FeatureVector fv) { - if (n == null) - return null; - else if (n.isLeafNode()) { - return n.getAllData(); - } else if (n.isDecisionNode()) { - Node next = ((DecisionNode) n).getNextNode(fv); - return interpret(next, fv); - } else if (n.isDirectedGraphNode()) { - DirectedGraphNode g = (DirectedGraphNode) n; - Object data = interpret(g.getDecisionNode(), fv); - if (data != null) { // OK, found something more specific - return data; - } - return interpret(g.getLeafNode(), fv); - } - throw new IllegalArgumentException("Unknown node type: " + n.getClass()); - } - - /** - * Return an iterator which returns all nodes in the tree exactly once. Search is done in a depth-first way. - * - * @return - */ - public Iterator getNodeIterator() { - return new NodeIterator(this, true, true, true); - } - - /** - * Return an iterator which returns all leaf nodes in the tree exactly once. Search is done in a depth-first way. - * - * @return - */ - public Iterator getLeafNodeIterator() { - return new NodeIterator(this, true, false, false); - } - - /** - * Return an iterator which returns all decision nodes in the tree exactly once. Search is done in a depth-first way. - * - * @return - */ - public Iterator getDecisionNodeIterator() { - return new NodeIterator(this, false, true, false); - } - - /** - * Return an iterator which returns all directed graph nodes in the tree exactly once. Search is done in a depth-first way. - * - * @return - */ - public Iterator getDirectedGraphNodeIterator() { - return new NodeIterator(this, false, false, true); - } - - /** - * A representation of the corresponding node iterator that can be used in extended for() statements. - * - * @return - */ - public Iterable getNodes() { - return new Iterable() { - public Iterator iterator() { - return getNodeIterator(); - } - }; - } - - /** - * A representation of the corresponding node iterator that can be used in extended for() statements. - * - * @return - */ - public Iterable getLeafNodes() { - return new Iterable() { - public Iterator iterator() { - return getLeafNodeIterator(); - } - }; - } - - /** - * A representation of the corresponding node iterator that can be used in extended for() statements. - * - * @return - */ - public Iterable getDecisionNodes() { - return new Iterable() { - public Iterator iterator() { - return getDecisionNodeIterator(); - } - }; - } - - /** - * A representation of the corresponding node iterator that can be used in extended for() statements. - * - * @return - */ - public Iterable getDirectedGraphNodes() { - return new Iterable() { - public Iterator iterator() { - return getDirectedGraphNodeIterator(); - } - }; - } - - /** - * Get the properties object associated with this tree, or null if there is no such object. - * - * @return - */ - public Properties getProperties() { - return properties; - } - - /** - * Get the root node of this CART - * - * @return the root node - */ - public Node getRootNode() { - return rootNode; - } - - /** - * Set the root node of this CART - * - * @param the - * root node - */ - public void setRootNode(Node rNode) { - rootNode = rNode; - } - - public FeatureDefinition getFeatureDefinition() { - return featDef; - } - - /** - * Get the number of nodes in this CART - * - * @return the number of nodes - */ - public int getNumNodes() { - if (rootNode == null) - return 0; - return rootNode.getNumberOfNodes(); - } - - public String toString() { - return this.rootNode.toString(""); - } - -} diff --git a/marytts-unitselection/src/main/java/marytts/cart/DirectedGraphNode.java b/marytts-unitselection/src/main/java/marytts/cart/DirectedGraphNode.java deleted file mode 100644 index 7190a7e723..0000000000 --- a/marytts-unitselection/src/main/java/marytts/cart/DirectedGraphNode.java +++ /dev/null @@ -1,218 +0,0 @@ -/** - * Copyright 2009 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ - -package marytts.cart; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import marytts.features.FeatureVector; - -/** - * A type of node that can be at the same time a decision node and a leaf node, and that can have more than one mother. Other than - * tree nodes, thus, directed graph nodes are not necessarily contained in a strict tree structure; furthermore, each node can - * potentially carry data. - * - * @author marc - * - */ -public class DirectedGraphNode extends Node { - - private DecisionNode decisionNode; - private Node leafNode; - - private Map motherToIndex = new HashMap(); - private List mothers = new ArrayList(); - private int uniqueID; - - /** - * - */ - public DirectedGraphNode(DecisionNode decisionNode, Node leafNode) { - setDecisionNode(decisionNode); - setLeafNode(leafNode); - } - - public DecisionNode getDecisionNode() { - return decisionNode; - } - - @Override - public boolean isDirectedGraphNode() { - return true; - } - - public void setDecisionNode(DecisionNode newNode) { - this.decisionNode = newNode; - if (newNode != null) - newNode.setMother(this, 0); - } - - public Node getLeafNode() { - return leafNode; - } - - public void setLeafNode(Node newNode) { - if (!(newNode == null || newNode instanceof DirectedGraphNode || newNode instanceof LeafNode)) { - throw new IllegalArgumentException("Only leaf nodes and directed graph nodes allowed as leafNode"); - } - this.leafNode = newNode; - if (newNode != null) - newNode.setMother(this, 0); - } - - @Override - public void setMother(Node node, int nodeIndex) { - mothers.add(node); - motherToIndex.put(node, nodeIndex); - } - - /** - * Get a mother node of this node. DirectedGraphNodes can have more than one node. - * - * @return the first mother, or null if there is no mother. - */ - public Node getMother() { - if (mothers.isEmpty()) - return null; - return mothers.get(0); - } - - /** - * Get the index of this node in the mother returned by getMother(). - * - * @return the index in the mother's daughter array, or 0 if there is no mother. - */ - public int getNodeIndex() { - Node firstMother = getMother(); - if (firstMother != null) - return motherToIndex.get(firstMother); - return 0; - } - - public List getMothers() { - return mothers; - } - - /** - * Return this node's index in the given mother's array of daughters. - * - * @param aMother - * @return - * @throws IllegalArgumentException - * if mother is not a mother of this node. - */ - public int getNodeIndex(Node aMother) { - if (!motherToIndex.containsKey(aMother)) - throw new IllegalArgumentException("The given node is not a mother of this node"); - return motherToIndex.get(aMother); - } - - /** - * Remove the given node from the list of mothers. - * - * @param aMother - * @throws IllegalArgumentException - * if mother is not a mother of this node. - */ - public void removeMother(Node aMother) { - if (!motherToIndex.containsKey(aMother)) - throw new IllegalArgumentException("The given node is not a mother of this node"); - motherToIndex.remove(aMother); - mothers.remove(aMother); - } - - @Override - protected void fillData(Object target, int pos, int len) { - if (leafNode != null) - leafNode.fillData(target, pos, len); - } - - @Override - public Object getAllData() { - if (leafNode != null) - return leafNode.getAllData(); - else if (decisionNode != null) - return decisionNode.getAllData(); - return null; - } - - @Override - public int getNumberOfData() { - if (leafNode != null) - return leafNode.getNumberOfData(); - else if (decisionNode != null) - return decisionNode.getNumberOfData(); - return 0; - } - - @Override - public int getNumberOfNodes() { - if (decisionNode != null) - return decisionNode.getNumberOfNodes(); - return 0; - } - - public Node getNextNode(FeatureVector fv) { - if (decisionNode != null) { - Node next = decisionNode.getNextNode(fv); - if (next != null) - return next; - } - return leafNode; - } - - public int getUniqueGraphNodeID() { - return uniqueID; - } - - public void setUniqueGraphNodeID(int id) { - this.uniqueID = id; - } - - public String getDecisionPath() { - StringBuilder ancestorInfo = new StringBuilder(); - if (getMothers().size() == 0) - ancestorInfo.append("null"); - for (Node mum : getMothers()) { - if (ancestorInfo.length() > 0) { - ancestorInfo.append(" or\n"); - } - if (mum.isDecisionNode()) { - ancestorInfo.append(((DecisionNode) mum).getDecisionPath(getNodeIndex())); - } else { - ancestorInfo.append(mum.getDecisionPath()); - } - } - return ancestorInfo + " - " + toString(); - } - - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append("DGN"); - sb.append(uniqueID); - if (motherToIndex.size() > 1) { - sb.append(" (").append(motherToIndex.size()).append(" mothers)"); - } - return sb.toString(); - } -} diff --git a/marytts-unitselection/src/main/java/marytts/cart/FeatureVectorCART.java b/marytts-unitselection/src/main/java/marytts/cart/FeatureVectorCART.java deleted file mode 100644 index bc7965fcf7..0000000000 --- a/marytts-unitselection/src/main/java/marytts/cart/FeatureVectorCART.java +++ /dev/null @@ -1,129 +0,0 @@ -/** - * Copyright 2007 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.cart; - -import java.io.IOException; - -import marytts.cart.impose.FeatureArrayIndexer; -import marytts.cart.impose.MaryNode; -import marytts.features.FeatureDefinition; -import marytts.features.FeatureVector; - -/** - * @author marc - * - */ -public class FeatureVectorCART extends CART { - - /** - * Convert the given Mary node tree into a CART with the leaves containing featureVectors - * - * @param tree - * the tree - * @param ffi - * the feature file indexer containing the feature vectors - */ - public FeatureVectorCART(MaryNode tree, FeatureArrayIndexer ffi) { - featDef = ffi.getFeatureDefinition(); - addDaughters(null, tree, ffi); - if (rootNode instanceof DecisionNode) { - ((DecisionNode) rootNode).countData(); - } - } - - public void load(String fileName, FeatureDefinition featDefinition, String[] setFeatureSequence) throws IOException { - throw new IllegalStateException("load() not implemented for FeatureVectorCART"); - } - - /** - * Add the given tree node as a daughter to the given mother node - * - * @param motherCARTNode - * the mother node - * @param currentTreeNode - * the tree node that we want to add - * @param ffi - * the feature file indexer containing the feature vectors - */ - private void addDaughters(DecisionNode motherCARTNode, MaryNode currentTreeNode, FeatureArrayIndexer ffi) { - if (currentTreeNode == null) { - LeafNode l = new LeafNode.FeatureVectorLeafNode(new FeatureVector[0]); - motherCARTNode.addDaughter(l); - return; - } - if (currentTreeNode.isNode()) { // if we are not at a leaf - - // System.out.print("Adding node, "); - // the next daughter - DecisionNode daughterNode = null; - // the number of daughters of the next daughter - int numDaughters; - // the index of the next feature - int nextFeatIndex = currentTreeNode.getFeatureIndex(); - // System.out.print("featureIndex = "+nextFeatIndex+"\n"); - if (featDef.isByteFeature(nextFeatIndex)) { - // if we have a byte feature, build a byte decision node - numDaughters = featDef.getNumberOfValues(nextFeatIndex); - daughterNode = new DecisionNode.ByteDecisionNode(nextFeatIndex, numDaughters, featDef); - } else { - if (featDef.isShortFeature(nextFeatIndex)) { - // if we have a short feature, build a short decision node - numDaughters = featDef.getNumberOfValues(nextFeatIndex); - daughterNode = new DecisionNode.ShortDecisionNode(nextFeatIndex, numDaughters, featDef); - } else { - // feature is of type float, currently not supported in ffi - throw new IllegalArgumentException("Found float feature in FeatureFileIndexer!"); - } - } - - if (motherCARTNode == null) { - // if the mother is null, the current node is the root - rootNode = daughterNode; - daughterNode.setIsRoot(true); - } else { - // if the current node is not the root, - // set mother and daughter accordingly - motherCARTNode.addDaughter(daughterNode); - } - // for every daughter go in recursion - for (int i = 0; i < numDaughters; i++) { - MaryNode nextChild = currentTreeNode.getChild(i); - addDaughters(daughterNode, nextChild, ffi); - - } - } else { - // we are at a leaf node - // System.out.println("Adding leaf"); - // get the feature vectors - FeatureVector[] featureVectors = ffi.getFeatureVectors(currentTreeNode.getFrom(), currentTreeNode.getTo()); - // build a new leaf - LeafNode leaf = new LeafNode.FeatureVectorLeafNode(featureVectors); - - if (motherCARTNode == null) { - // if the mother is null, the current node is the root - rootNode = leaf; - } else { - // set mother and daughter - motherCARTNode.addDaughter(leaf); - } - } - } - -} diff --git a/marytts-unitselection/src/main/java/marytts/cart/LeafNode.java b/marytts-unitselection/src/main/java/marytts/cart/LeafNode.java deleted file mode 100644 index 44614de7ab..0000000000 --- a/marytts-unitselection/src/main/java/marytts/cart/LeafNode.java +++ /dev/null @@ -1,541 +0,0 @@ -/** - * Copyright 2000-2009 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.cart; - -import java.util.ArrayList; -import java.util.List; - -import marytts.exceptions.MaryConfigurationException; -import marytts.features.FeatureDefinition; -import marytts.features.FeatureVector; - -/** - * The leaf of a CART. - */ -public abstract class LeafNode extends Node { - - public enum LeafType { - IntArrayLeafNode, FloatLeafNode, IntAndFloatArrayLeafNode, StringAndFloatLeafNode, FeatureVectorLeafNode, PdfLeafNode - }; - - // unique index used in MaryCART format - protected int uniqueLeafId; - - /** - * Create a new LeafNode. - * - * @param tok - * the String Tokenizer containing the String with the indices - * @param openBrackets - * the number of opening brackets at the first token - */ - public LeafNode() { - super(); - isRoot = false; - } - - @Override - public boolean isLeafNode() { - return true; - } - - /** - * Count all the nodes at and below this node. A leaf will return 1; the root node will report the total number of decision - * and leaf nodes in the tree. - * - * @return - */ - public int getNumberOfNodes() { - return 1; - } - - // unique index used in MaryCART format - public void setUniqueLeafId(int id) { - this.uniqueLeafId = id; - } - - public int getUniqueLeafId() { - return uniqueLeafId; - } - - public String toString() { - return "id" + uniqueLeafId; - } - - /** - * Indicate whether the leaf node has no meaningful data. - * - * @return - */ - public abstract boolean isEmpty(); - - /** - * Count all the data available at and below this node. The meaning of this depends on the type of nodes; for example, when - * IntArrayLeafNodes are used, it is the total number of ints that are saved in all leaf nodes below the current node. - * - * @return an int counting the data below the current node, or -1 if such a concept is not meaningful. - */ - public abstract int getNumberOfData(); - - /** - * Get all the data at or below this node. The type of data returned depends on the type of nodes; for example, when - * IntArrayLeafNodes are used, one int[] is returned which contains all int values in all leaf nodes below the current node. - * - * @return an object containing all data below the current node, or null if such a concept is not meaningful. - */ - public abstract Object getAllData(); - - /** - * Write this node's data into the target object at pos, making sure that exactly len data are written. The type of data - * written depends on the type of nodes; for example, when IntArrayLeafNodes are used, target would be an int[]. - * - * @param array - * the object to write to, usually an array. - * @param pos - * the position in the target at which to start writing - * @param len - * the amount of data items to write, usually equals getNumberOfData(). - */ - protected abstract void fillData(Object target, int pos, int len); - - /** - * The type of this leaf node. - * - * @return - */ - public abstract LeafType getLeafNodeType(); - - /** - * An LeafNode class suitable for representing the leaves of classification trees -- the leaf is a collection of items - * identified by an index number. - * - * @author marc - * - */ - public static class IntArrayLeafNode extends LeafNode { - protected int[] data; - - public IntArrayLeafNode(int[] data) { - super(); - this.data = data; - } - - /** - * Get all data in this leaf - * - * @return the int array contained in this leaf - */ - public Object getAllData() { - return data; - } - - public int[] getIntData() { - return data; - } - - protected void fillData(Object target, int pos, int len) { - if (!(target instanceof int[])) - throw new IllegalArgumentException("Expected target object of type int[], got " + target.getClass()); - int[] array = (int[]) target; - assert len <= data.length; - System.arraycopy(data, 0, array, pos, len); - } - - public int getNumberOfData() { - if (data != null) - return data.length; - return 0; - } - - public boolean isEmpty() { - return data == null || data.length == 0; - } - - public LeafType getLeafNodeType() { - return LeafType.IntArrayLeafNode; - } - - public String toString() { - if (data == null) - return super.toString() + "(int[null])"; - return super.toString() + "(int[" + data.length + "])"; - } - - } - - public static class IntAndFloatArrayLeafNode extends IntArrayLeafNode { - - protected float[] floats; - - public IntAndFloatArrayLeafNode(int[] data, float[] floats) { - super(data); - this.floats = floats; - } - - public float[] getFloatData() { - return floats; - } - - /** - * For the int-float pairs in this leaf, return the int value for which the associated float value is the highest one. If - * the float values are probabilities, this method returns the most probable int. - * - * @return - */ - public int mostProbableInt() { - int bestInd = 0; - float maxProb = 0f; - - for (int i = 0; i < data.length; i++) { - if (floats[i] > maxProb) { - maxProb = floats[i]; - bestInd = data[i]; - } - } - return bestInd; - } - - /** - * Delete a candidate of the leaf by its given data/index - * - * @param target - * the given data - */ - public void eraseData(int target) { - int[] newData = new int[data.length - 1]; - float[] newFloats = new float[floats.length - 1]; - int index = 0; - for (int i = 0; i < data.length; i++) { - if (data[i] != target) { - newData[index] = data[i]; - newFloats[index] = floats[i]; - index++; - } - } - data = newData; - floats = newFloats; - } - - public LeafType getLeafNodeType() { - return LeafType.IntAndFloatArrayLeafNode; - } - - public String toString() { - if (data == null) - return super.toString() + "(int and floats[null])"; - return super.toString() + "(int and floats[" + data.length + "])"; - } - - } - - public static class StringAndFloatLeafNode extends IntAndFloatArrayLeafNode { - - public StringAndFloatLeafNode(int[] data, float[] floats) { - super(data, floats); - } - - /** - * Return the most probable value in this leaf, translated into its string representation using the featureIndex'th - * feature of the given feature definition. - * - * @param featureDefinition - * @param featureIndex - * @return - */ - public String mostProbableString(FeatureDefinition featureDefinition, int featureIndex) { - int bestInd = mostProbableInt(); - return featureDefinition.getFeatureValueAsString(featureIndex, bestInd); - } - - public String toString() { - if (data == null) - return super.toString() + "(string and floats[null])"; - return super.toString() + "(string and floats[" + data.length + "])"; - } - - public LeafType getLeafNodeType() { - return LeafType.StringAndFloatLeafNode; - } - - } - - public static class FeatureVectorLeafNode extends LeafNode { - private FeatureVector[] featureVectors; - private List featureVectorList; - private boolean growable; - - /** - * Build a new leaf node containing the given feature vectors - * - * @param featureVectors - * the feature vectors - */ - public FeatureVectorLeafNode(FeatureVector[] featureVectors) { - super(); - this.featureVectors = featureVectors; - growable = false; - } - - /** - * Build a new, empty leaf node to be filled with vectors later - * - */ - public FeatureVectorLeafNode() { - super(); - featureVectorList = new ArrayList(); - featureVectors = null; - growable = true; - } - - public void addFeatureVector(FeatureVector fv) { - featureVectorList.add(fv); - } - - /** - * Get the feature vectors of this node - * - * @return the feature vectors - */ - public FeatureVector[] getFeatureVectors() { - if (growable && (featureVectors == null || featureVectors.length == 0)) { - featureVectors = (FeatureVector[]) featureVectorList.toArray(new FeatureVector[featureVectorList.size()]); - } - return featureVectors; - } - - public void setFeatureVectors(FeatureVector[] fv) { - this.featureVectors = fv; - } - - /** - * Get all data in this leaf - * - * @return the featurevector array contained in this leaf - */ - public Object getAllData() { - if (growable && (featureVectors == null || featureVectors.length == 0)) { - featureVectors = (FeatureVector[]) featureVectorList.toArray(new FeatureVector[featureVectorList.size()]); - } - return featureVectors; - } - - protected void fillData(Object target, int pos, int len) { - if (!(target instanceof FeatureVector[])) - throw new IllegalArgumentException("Expected target object of type FeatureVector[], got " + target.getClass()); - FeatureVector[] array = (FeatureVector[]) target; - assert len <= featureVectors.length; - System.arraycopy(featureVectors, 0, array, pos, len); - } - - public int getNumberOfData() { - if (growable) { - return featureVectorList.size(); - } - if (featureVectors != null) - return featureVectors.length; - return 0; - } - - public boolean isEmpty() { - return featureVectors == null || featureVectors.length == 0; - } - - public LeafType getLeafNodeType() { - return LeafType.FeatureVectorLeafNode; - } - - public String toString() { - if (growable) - return "fv[" + featureVectorList.size() + "]"; - if (featureVectors == null) - return super.toString() + "(fv[null])"; - return super.toString() + "(fv[" + featureVectors.length + "])"; - } - - } - - /** - * A leaf class that is suitable for regression trees. Here, a leaf consists of a mean and a standard deviation. - * - * @author marc - * - */ - public static class FloatLeafNode extends LeafNode { - private float[] data; - - public FloatLeafNode(float[] data) { - super(); - if (data.length != 2) - throw new IllegalArgumentException("data must have length 2, found " + data.length); - this.data = data; - } - - /** - * Get all data in this leaf - * - * @return the mean/standard deviation value contained in this leaf - */ - public Object getAllData() { - return data; - } - - public int getDataLength() { - return data.length; - } - - public float getMean() { - return data[1]; - } - - public float getStDeviation() { - return data[0]; - } - - protected void fillData(Object target, int pos, int len) { - throw new IllegalStateException("This method should not be called for FloatLeafNodes"); - } - - public int getNumberOfData() { - return 1; - } - - public boolean isEmpty() { - return false; - } - - public LeafType getLeafNodeType() { - return LeafType.FloatLeafNode; - } - - public String toString() { - if (data == null) - return super.toString() + "(mean=null, stddev=null)"; - return super.toString() + "(mean=" + data[1] + ", stddev=" + data[0] + ")"; - } - - } - - /** - * A leaf class that is suitable for regression trees. Here, a leaf consists of a mean and a diagonal covariance vectors. This - * node will be used in HTS CART trees. - * - */ - public static class PdfLeafNode extends LeafNode { - private int vectorSize; - private double[] mean; // mean vector. - private double[] variance; // diagonal covariance. - private double voicedWeight; // only for lf0 tree. - - /** - * @param idx - * , a unique index number - * @param pdf - * , pdf[numStreams][2*vectorSize] - */ - public PdfLeafNode(int idx, double pdf[][]) throws MaryConfigurationException { - super(); - this.setUniqueLeafId(idx); - // System.out.println("adding leaf node: " + idx); - if (pdf != null) { - double val; - int i, j, vsize, nstream; - nstream = pdf.length; - - if (nstream == 1) { // This is the case for dur, mgc, str, mag, or joinModel. - vsize = (pdf[0].length) / 2; - vectorSize = vsize; - mean = new double[vsize]; - for (i = 0, j = 0; j < vsize; i++, j++) - mean[i] = pdf[0][j]; - variance = new double[vsize]; - for (i = 0, j = vsize; j < (2 * vsize); i++, j++) - variance[i] = pdf[0][j]; - - } else { // this is the case for lf0 - vectorSize = nstream; - mean = new double[nstream]; - variance = new double[nstream]; - for (int stream = 0; stream < nstream; stream++) { - mean[stream] = pdf[stream][0]; - variance[stream] = pdf[stream][1]; - // vw = lf0pdf[numStates][numPdfs][numStreams][2]; /* voiced weight */ - // uvw = lf0pdf[numStates][numPdfs][numStreams][3]; /* unvoiced weight */ - if (stream == 0) - voicedWeight = pdf[stream][2]; - } - } - } else { - throw new MaryConfigurationException("PdfLeafNode: pdf vector is null for index=" + idx); - } - - } - - public int getDataLength() { - return mean.length; - } - - public double[] getMean() { - return mean; - } - - public double[] getVariance() { - return variance; - } - - public double getVoicedWeight() { - return voicedWeight; - } - - public int getVectorSize() { - return vectorSize; - } - - protected void fillData(Object target, int pos, int len) { - throw new IllegalStateException("This method should not be called for PdfLeafNodes"); - } - - // not meaningful here. - public Object getAllData() { - return null; - } - - // not meaningful here. - // i need this value positive when searching ??? - public int getNumberOfData() { - return 1; - } - - public boolean isEmpty() { - return false; - } - - public LeafType getLeafNodeType() { - return LeafType.PdfLeafNode; - } - - public String toString() { - if (mean == null) - return super.toString() + "(mean=null, stddev=null)"; - return super.toString() + "(mean=[" + vectorSize + "], stddev=[" + vectorSize + "])"; - } - - } - -} diff --git a/marytts-unitselection/src/main/java/marytts/cart/Node.java b/marytts-unitselection/src/main/java/marytts/cart/Node.java deleted file mode 100644 index 4ebf91f571..0000000000 --- a/marytts-unitselection/src/main/java/marytts/cart/Node.java +++ /dev/null @@ -1,172 +0,0 @@ -/** - * Portions Copyright 2006 DFKI GmbH. - * Portions Copyright 2001 Sun Microsystems, Inc. - * Portions Copyright 1999-2001 Language Technologies Institute, - * Carnegie Mellon University. - * All Rights Reserved. Use is subject to license terms. - * - * Permission is hereby granted, free of charge, to use and distribute - * this software and its documentation without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of this work, and to - * permit persons to whom this work is furnished to do so, subject to - * the following conditions: - * - * 1. The code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * 2. Any modifications must be clearly marked as such. - * 3. Original authors' names are not deleted. - * 4. The authors' names are not used to endorse or promote products - * derived from this software without specific prior written - * permission. - * - * DFKI GMBH AND THE CONTRIBUTORS TO THIS WORK DISCLAIM ALL WARRANTIES WITH - * REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL DFKI GMBH NOR THE - * CONTRIBUTORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL - * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR - * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS - * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF - * THIS SOFTWARE. - */ -package marytts.cart; - -/** - * A node for the CART or DirectedGraph. All node types inherit from this class - */ -public abstract class Node { - // isRoot should be set to true if this node is the root node - protected boolean isRoot; - - // every node except the root node has a mother - protected Node mother; - - // the index of the node in the daughters array of its mother - protected int nodeIndex; - - /** - * set the mother node of this node, and remember this node's index in mother. - * - * @param node - * the mother node - * @param nodeIndex - * the index of this node in the mother node's list of daughters - */ - public void setMother(Node node, int nodeIndex) { - this.mother = node; - this.nodeIndex = nodeIndex; - } - - /** - * Get the mother node of this node - * - * @return the mother node - */ - public Node getMother() { - return mother; - } - - /** - * Get the index of this node in the mother's array of daughters - * - * @return the index - */ - public int getNodeIndex() { - return nodeIndex; - } - - /** - * Set isRoot to the given value - * - * @param isRoot - * the new value of isRoot - */ - public void setIsRoot(boolean isRoot) { - this.isRoot = isRoot; - } - - /** - * Get the setting of isRoot - * - * @return the setting of isRoot - */ - public boolean isRoot() { - return isRoot; - } - - public boolean isDecisionNode() { - return false; - } - - public boolean isLeafNode() { - return false; - } - - public boolean isDirectedGraphNode() { - return false; - } - - public Node getRootNode() { - if (isRoot) { - assert mother == null; - return this; - } else { - assert mother != null : " I am not root but I have no mother :-("; - return mother.getRootNode(); - } - } - - public String getDecisionPath() { - String ancestorInfo; - if (mother == null) - ancestorInfo = "null"; - else if (mother.isDecisionNode()) { - ancestorInfo = ((DecisionNode) mother).getDecisionPath(getNodeIndex()); - } else { - ancestorInfo = mother.getDecisionPath(); - } - return ancestorInfo + " - " + toString(); - } - - /** - * Count all the nodes at and below this node. A leaf will return 1; the root node will report the total number of decision - * and leaf nodes in the tree. - * - * @return - */ - public abstract int getNumberOfNodes(); - - /** - * Count all the data available at and below this node. The meaning of this depends on the type of nodes; for example, when - * IntArrayLeafNodes are used, it is the total number of ints that are saved in all leaf nodes below the current node. - * - * @return an int counting the data below the current node, or -1 if such a concept is not meaningful. - */ - public abstract int getNumberOfData(); - - /** - * Get all the data at or below this node. The type of data returned depends on the type of nodes; for example, when - * IntArrayLeafNodes are used, one int[] is returned which contains all int values in all leaf nodes below the current node. - * - * @return an object containing all data below the current node, or null if such a concept is not meaningful. - */ - public abstract Object getAllData(); - - /** - * Write this node's data into the target object at pos, making sure that exactly len data are written. The type of data - * written depends on the type of nodes; for example, when IntArrayLeafNodes are used, target would be an int[]. - * - * @param array - * the object to write to, usually an array. - * @param pos - * the position in the target at which to start writing - * @param len - * the amount of data items to write, usually equals getNumberOfData(). - */ - protected abstract void fillData(Object target, int pos, int len); - - public String toString(String prefix) { - return prefix + this.toString(); - } - -} \ No newline at end of file diff --git a/marytts-unitselection/src/main/java/marytts/cart/NodeIterator.java b/marytts-unitselection/src/main/java/marytts/cart/NodeIterator.java deleted file mode 100644 index f3a462f7ca..0000000000 --- a/marytts-unitselection/src/main/java/marytts/cart/NodeIterator.java +++ /dev/null @@ -1,163 +0,0 @@ -/** - * Copyright 2009 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ - -package marytts.cart; - -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.Map; -import java.util.Set; - -/** - * @author marc - * - */ -public class NodeIterator implements Iterator { - private Node root; - private Node current; - private boolean showLeafNodes; - private boolean showDecisionNodes; - private boolean showDirectedGraphNodes; - private Set alreadySeen = new HashSet(); - // we need to keep our own map of daughter-mother relationships, - // because for subgraphs, we could move out of a subgraph if we call node.getMother() - // if the mother via which we entered a multi-parent node is not the first mother. - private Map daughterToMother = new HashMap(); - - /** - * Iterate over all nodes in the graph. - * - * @param graph - * @param showLeafNodes - * @param showDecisionNodes - * @param showDirectedGraphNodes - */ - protected NodeIterator(DirectedGraph graph, boolean showLeafNodes, boolean showDecisionNodes, boolean showDirectedGraphNodes) { - this(graph.getRootNode(), showLeafNodes, showDecisionNodes, showDirectedGraphNodes); - } - - /** - * Iterate over the subtree below rootNode. - * - * @param rootNode - * @param showLeafNodes - * @param showDecisionNodes - * @param showDirectedGraphNodes - */ - protected NodeIterator(Node rootNode, boolean showLeafNodes, boolean showDecisionNodes, boolean showDirectedGraphNodes) { - this.root = rootNode; - this.showLeafNodes = showLeafNodes; - this.showDecisionNodes = showDecisionNodes; - this.showDirectedGraphNodes = showDirectedGraphNodes; - this.current = root; - alreadySeen.add(current); - if (!currentIsSuitable()) { - nextSuitableNodeDepthFirst(); - } - } - - public boolean hasNext() { - return current != null; - } - - public T next() { - T ret = (T) current; - // and already prepare the current one - nextSuitableNodeDepthFirst(); - return ret; - } - - private boolean currentIsSuitable() { - return (current == null || showDecisionNodes && current.isDecisionNode() || showLeafNodes && current.isLeafNode() || showDirectedGraphNodes - && current.isDirectedGraphNode()); - } - - private void nextSuitableNodeDepthFirst() { - do { - nextNodeDepthFirst(); - } while (!currentIsSuitable()); - } - - private void nextNodeDepthFirst() { - if (current == null) - return; - if (current.isDecisionNode()) { - DecisionNode dec = (DecisionNode) current; - for (int i = 0; i < dec.getNumberOfDaugthers(); i++) { - Node daughter = dec.getDaughter(i); - if (daughter == null) - continue; - daughterToMother.put(daughter, dec); - if (unseenNode(dec.getDaughter(i))) - return; - } - } else if (current.isDirectedGraphNode()) { - // Graph nodes return leaf child first, then decision child - DirectedGraphNode g = (DirectedGraphNode) current; - Node leaf = g.getLeafNode(); - if (leaf != null) { - daughterToMother.put(leaf, g); - if (unseenNode(leaf)) - return; - } - Node dec = g.getDecisionNode(); - if (dec != null) { - daughterToMother.put(dec, g); - if (unseenNode(dec)) - return; - } - } - // If we didn't find a suitable child, we need to: - backtrace(); - } - - private void backtrace() { - // Only go back to mothers we have come from. - // This has two effects: - // 1. We cannot go beyond root node; - // 2. we don't risk to leave the subgraph defined by root node - // in cases where we enter into a multi-parent node from a not-first mother - // (in such cases, getMother() would return the first mother). - current = daughterToMother.get(current); - nextNodeDepthFirst(); - } - - /** - * Test whether the given node is unseen. If so, move current to it, and remember it as a seen node. - * - * @param candidate - * @return - */ - private boolean unseenNode(Node candidate) { - if (candidate != null && !alreadySeen.contains(candidate)) { - current = candidate; - alreadySeen.add(current); - return true; - } - return false; - - } - - public void remove() { - throw new UnsupportedOperationException("Cannot remove nodes using this iterator"); - } - -} diff --git a/marytts-unitselection/src/main/java/marytts/cart/StringPredictionTree.java b/marytts-unitselection/src/main/java/marytts/cart/StringPredictionTree.java deleted file mode 100644 index 8f9c7c902a..0000000000 --- a/marytts-unitselection/src/main/java/marytts/cart/StringPredictionTree.java +++ /dev/null @@ -1,222 +0,0 @@ -/** - * Copyright 2000-2009 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.cart; - -import java.io.BufferedReader; -import java.io.IOException; -import java.util.regex.Pattern; - -import marytts.cart.LeafNode.IntAndFloatArrayLeafNode; -import marytts.cart.LeafNode.LeafType; -import marytts.cart.io.WagonCARTReader; -import marytts.features.FeatureDefinition; -import marytts.features.FeatureVector; -import marytts.features.Target; - -//import com.sun.tools.javac.code.Attribute.Array; - -//public class StringPredictionTree extends ExtendedClassificationTree { -public class StringPredictionTree extends CART { - public static final String ENC_LINE_START = ";;target={"; - public static final String ENC_LINE_END = "}\n"; - - // TODO: maybe use an HashMap - // this strores the strings that correspond to the indices at the leaves - String[] stringIdDecoding; - Pattern splitPattern = Pattern.compile("'"); - Pattern delimPattern = Pattern.compile(",\\d+:|}$"); - - /** - * - * @param rootNode - * the root node of this tree. This node has to be set to be a root node beforehand. - * @param featDef - * the featureDefinition used in this tree - * - * @author ben - */ - public StringPredictionTree(Node aRootNode, FeatureDefinition aFeatDef, String[] aTargetDecoding) { - if (!aRootNode.isRoot()) - throw new IllegalArgumentException("Tried to set a non-root-node as root of the tree. "); - - this.rootNode = aRootNode; - this.featDef = aFeatDef; - this.stringIdDecoding = aTargetDecoding; - } - - /** - * - * This constructs a new string prediciton tree from a stream containing a tree in wagon format. In addition to the - * constructor of ExtendedClassificationTree it reads in the mapping from numbers to the Strings from a stream. The encoding - * has to be the first line in the file (a empty line is allowed). - * - * It has the form: - * - * ;;target={1:'string_a',2:'string_b,'...',26:'string_z'} - * - */ - - public StringPredictionTree(BufferedReader reader, FeatureDefinition featDefinition) throws IOException { - - String line = reader.readLine(); - - if (line.equals("")) {// first line is empty, read again - line = reader.readLine(); - } - - if (line.startsWith(ENC_LINE_START)) { - - // split of the beginning of the string - String rawLine = line.substring((ENC_LINE_START + "0:'").length()); - - // regular expression for splitting of the target encodings - // ',NUMBER:' OR '} - - String[] splitted = splitPattern.split(rawLine); - - this.stringIdDecoding = new String[splitted.length / 2]; - - for (int i = 0; i < splitted.length / 2; i++) { - this.stringIdDecoding[i] = splitted[i * 2]; - if (!this.delimPattern.matcher(splitted[i * 2 + 1]).matches()) { - throw new IllegalArgumentException("wrong encoding for the mapping of numbers and strings."); - } - } - - // System.err.println(rawLine); - // System.err.println(Arrays.toString(stringIdDecoding)); - - // encoding/linebreak problems with this line? - // this.stringIdDecoding = rawLine.split("',\\d+:'|'}$"); - - } else - throw new IllegalArgumentException("First line must be a comment line specifying the target symbols."); - - // read the rest of the tree - // old: this.load(reader, featDefinition); - // CHECK!! this has not been tested, maybe it does not work!!! - WagonCARTReader wagonReader = new WagonCARTReader(LeafType.IntAndFloatArrayLeafNode); - this.setRootNode(wagonReader.load(reader, featDefinition)); - - } - - // toString method, that writes the decoding in first line, - // should be something like: - // ;;target={1:'string_a',2:'string_b',...,26:'string_z'} - // this is followed by a - public String toString() { - - // make String representation of target symbol decoding and invoke super-toString - StringBuilder sb = new StringBuilder(); - - sb.append(ENC_LINE_START); - - for (int i = 0; i < this.stringIdDecoding.length; i++) { - - if (i > 0) - sb.append(","); - - sb.append(i); - sb.append(":'"); - sb.append(this.stringIdDecoding[i]); - sb.append("'"); - } - - sb.append(ENC_LINE_END); - sb.append(super.toString()); - - return sb.toString(); - } - - /** - * TODO: copied from CART, does not work as expected with minNumberOfData = 0 - * - * Passes the given item through this CART and returns the leaf Node, or the Node it stopped walking down. - * - * @param target - * the target to analyze - * @param minNumberOfData - * the minimum number of data requested. If this is 0, walk down the CART until the leaf level. - * - * @return the Node - */ - public Node interpretToNode(FeatureVector featureVector, int minNumberOfData) { - Node currentNode = rootNode; - Node prevNode = null; - - // logger.debug("Starting cart at "+nodeIndex); - while (currentNode.getNumberOfData() > minNumberOfData && !(currentNode instanceof LeafNode)) { - // while we have not reached the bottom, - // get the next node based on the features of the target - prevNode = currentNode; - currentNode = ((DecisionNode) currentNode).getNextNode(featureVector); - // logger.debug(decision.toString() + " result '"+ - // decision.findFeature(item) + "' => "+ nodeIndex); - } - - // Now usually we will have gone down one level too far - if (currentNode.getNumberOfData() < minNumberOfData && prevNode != null) { - currentNode = prevNode; - } - - assert currentNode.getNumberOfData() >= minNumberOfData || currentNode == rootNode; - - return currentNode; - - } - - public String getMostProbableString(FeatureVector aFV) { - - // get the node data - // TODO: for some reason, when I changed interpretToNode in taking a fv, I had to change mindata to -1 ?! - IntAndFloatArrayLeafNode predictedNode = (IntAndFloatArrayLeafNode) this.interpretToNode(aFV, -1); - - // look for the index with highest associated probability - float[] probs = predictedNode.getFloatData(); - int[] indices = predictedNode.getIntData(); - - int bestInd = 0; - float maxProb = 0f; - - for (int i = 0; i < indices.length; i++) { - if (probs[i] > maxProb) { - maxProb = probs[i]; - bestInd = indices[i]; - } - } - - if (bestInd >= stringIdDecoding.length) { - logger.info("looking up most probable string for feature vector"); - logger.error("index bigger than number of targets"); - logger.info("biggest index is " + (stringIdDecoding.length - 1) + "with the symbol" - + stringIdDecoding[stringIdDecoding.length - 1]); - } - - // get the String representation - return this.stringIdDecoding[bestInd]; - } - - public String getMostProbableString(Target aTarget) { - // get the String representation - return this.getMostProbableString(aTarget.getFeatureVector()); - - } - -} diff --git a/marytts-unitselection/src/main/java/marytts/cart/impose/FeatureArrayIndexer.java b/marytts-unitselection/src/main/java/marytts/cart/impose/FeatureArrayIndexer.java deleted file mode 100644 index d6a693d064..0000000000 --- a/marytts-unitselection/src/main/java/marytts/cart/impose/FeatureArrayIndexer.java +++ /dev/null @@ -1,422 +0,0 @@ -/** - * Copyright 2006 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.cart.impose; - -import java.util.Arrays; - -import marytts.features.FeatureDefinition; -import marytts.features.FeatureVector; - -/** - * A class branched from FeatureFileIndexer which works directly on a feature array, rather than extending FeatureFileReader. - * - * @author Marc Schröder - * - */ -public class FeatureArrayIndexer { - - private MaryNode tree = null; - private int[] featureSequence = null; - private FeatureComparator c = new FeatureComparator(-1, null); - private UnitIndexComparator cui = new UnitIndexComparator(); - private FeatureVector[] featureVectors; - private FeatureDefinition featureDefinition; - - private long numberOfLeaves = 0; - - /****************/ - /* CONSTRUCTORS */ - /****************/ - - /** - * Constructor which takes an array of feature vectors and launches an indexing operation according to a feature sequence - * constraint. - * - * @param featureVectors - * an array of feature vectors - * @param featureDefinition - * a feature definition to make sense of the feature vectors - * @param setFeatureSequence - * An array of indexes indicating the hierarchical order (or, equivalently, the sequence) of the features to use - * for the indexing. - * - */ - public FeatureArrayIndexer(FeatureVector[] featureVectors, FeatureDefinition featureDefinition, int[] setFeatureSequence) { - this(featureVectors, featureDefinition); - deepSort(setFeatureSequence); - } - - /** - * Constructor which takes an array of feature vectors and launches an indexing operation according to a feature sequence - * constraint. - * - * @param featureVectors - * an array of feature vectors - * @param featureDefinition - * a feature definition to make sense of the feature vectors - * @param setFeatureSequence - * An array of feature names indicating the hierarchical order (or, equivalently, the sequence) of the features to - * use for the indexing. - * - */ - public FeatureArrayIndexer(FeatureVector[] featureVectors, FeatureDefinition featureDefinition, String[] setFeatureSequence) { - this(featureVectors, featureDefinition); - deepSort(setFeatureSequence); - } - - /** - * Constructor which loads the feature vector array but does not launch an indexing operation. - * - * @param featureVectors - * an array of feature vectors - * @param featureDefinition - * a feature definition to make sense of the feature vectors - * - */ - public FeatureArrayIndexer(FeatureVector[] featureVectors, FeatureDefinition featureDefinition) { - this.featureVectors = featureVectors; - this.featureDefinition = featureDefinition; - } - - /********************/ - /* INDEXING METHODS */ - /********************/ - - /** - * A local sort at a particular node along the deep sorting operation. This is a recursive function. - * - * @param currentFeatureIdx - * The currently tested feature. - * @param currentNode - * The current node, holding the currently processed zone in the array of feature vectors. - */ - private void sortNode(int currentFeatureIdx, MaryNode currentNode) { - /* If we have reached a leaf, do a final sort according to the unit index and return: */ - if (currentFeatureIdx == featureSequence.length) { - Arrays.sort(featureVectors, currentNode.from, currentNode.to, cui); - numberOfLeaves++; - /* - * System.out.print( "LEAF ! (" + (currentNode.to-currentNode.from) + " units)" ); for ( int i = currentNode.from; i < - * currentNode.to; i++ ) { System.out.print( " (" + featureVectors[i].getUnitIndex() + " 0)" ); } System.out.println( - * "" ); - */ - return; - } - /* Else: */ - int currentFeature = featureSequence[currentFeatureIdx]; - FeatureVector.FeatureType featureType = featureVectors[0].getFeatureType(currentFeature); - /* Register the feature currently used for the splitting */ - currentNode.setFeatureIndex(currentFeature); - /* Perform the sorting according to the currently considered feature: */ - /* 1) position the comparator onto the right feature */ - c.setFeatureIdx(currentFeature, featureType); - /* 2) do the sorting */ - Arrays.sort(featureVectors, currentNode.from, currentNode.to, c); - - /* - * Then, seek for the zones where the feature value is the same, and launch the next sort level on these. - */ - int nVal = featureDefinition.getNumberOfValues(currentFeature); - currentNode.split(nVal); - int nextFrom = currentNode.from; - int nextTo = currentNode.from; - for (int i = 0; i < nVal; i++) { - nextFrom = nextTo; - // System.out.print( "Next node begins at " + nextFrom ); - while ((nextTo < currentNode.to) && (featureVectors[nextTo].getFeatureAsInt(currentFeature) == i)) { - // System.out.print( " " + featureVectors[nextTo].getFeatureAsInt( currentFeature ) ); - nextTo++; - } - // System.out.println( " and ends at " + nextTo + " for a total of " + (nextTo-nextFrom) + " units." ); - if ((nextTo - nextFrom) != 0) { - MaryNode nod = new MaryNode(nextFrom, nextTo); - currentNode.setChild(i, nod); - // System.out.print("(" + i + " isByteOf " + currentFeature + ")" ); - sortNode(currentFeatureIdx + 1, nod); - } else - currentNode.setChild(i, null); - } - } - - /** - * Launches a deep sort on the array of feature vectors. This is public because it can be used to re-index the previously read - * feature file. - * - * @param featureIdx - * An array of feature indexes, indicating the sequence of features according to which the sorting should be - * performed. - */ - public void deepSort(int[] setFeatureSequence) { - featureSequence = setFeatureSequence; - numberOfLeaves = 0; - tree = new MaryNode(0, featureVectors.length); - sortNode(0, tree); - } - - /** - * Launches a deep sort on the array of feature vectors. This is public because it can be used to re-index the previously read - * feature file. - * - * @param featureIdx - * An array of feature names, indicating the sequence of features according to which the sorting should be - * performed. - */ - public void deepSort(String[] setFeatureSequence) { - featureSequence = featureDefinition.getFeatureIndexArray(setFeatureSequence); - numberOfLeaves = 0; - tree = new MaryNode(0, featureVectors.length); - sortNode(0, tree); - } - - /** - * Fill a particular node of a pre-specified tree. This is a recursive function. - * - * @param currentNode - * The current node, holding the currently processed zone in the array of feature vectors. - */ - private void fillNode(MaryNode currentNode) { - /* If we have reached a leaf, do a final sort according to the unit index and return: */ - if (currentNode.isLeaf()) { - Arrays.sort(featureVectors, currentNode.from, currentNode.to, cui); - numberOfLeaves++; - /* - * System.out.print( "LEAF ! (" + (currentNode.to-currentNode.from) + " units)" ); for ( int i = currentNode.from; i < - * currentNode.to; i++ ) { System.out.print( " (" + featureVectors[i].getUnitIndex() + " 0)" ); } System.out.println( - * "" ); - */ - return; - } - /* Else: */ - int currentFeature = currentNode.featureIndex; - FeatureVector.FeatureType featureType = featureVectors[0].getFeatureType(currentFeature); - /* Perform the sorting according to the currently considered feature: */ - /* 1) position the comparator onto the right feature */ - c.setFeatureIdx(currentFeature, featureType); - /* 2) do the sorting */ - Arrays.sort(featureVectors, currentNode.from, currentNode.to, c); - - /* - * Then, seek for the zones where the feature value is the same, and launch the next sort level on these. - */ - int nVal = featureDefinition.getNumberOfValues(currentFeature); - int nextFrom = currentNode.from; - int nextTo = currentNode.from; - for (int i = 0; i < nVal; i++) { - nextFrom = nextTo; - // System.out.print( "Next node begins at " + nextFrom ); - while ((nextTo < currentNode.to) && (featureVectors[nextTo].getFeatureAsInt(currentFeature) == i)) { - // System.out.print( " " + featureVectors[nextTo].getFeatureAsInt( currentFeature ) ); - nextTo++; - } - // System.out.println( " and ends at " + nextTo + " for a total of " + (nextTo-nextFrom) + " units." ); - if ((nextTo - nextFrom) != 0) { - MaryNode nod = currentNode.getChild(i); - if (nod != null) { - nod.from = nextFrom; - nod.to = nextTo; - fillNode(nod); - } - } else - currentNode.setChild(i, null); - } - } - - /** - * Fill a tree which specifies a feature hierarchy but no corresponding units. - * - * @param featureIdx - * An array of feature indexes, indicating the sequence of features according to which the sorting should be - * performed. - */ - public void deepFill(MaryNode specTree) { - tree = specTree; - numberOfLeaves = 0; - sortNode(0, tree); - } - - /***************************/ - /* QUERY/RETRIEVAL METHODS */ - /***************************/ - - /** - * Retrieve an array of unit features which complies with a specific target specification, according to an underlying tree. - * - * @param v - * A feature vector for which to send back an array of complying unit indexes. - * @return A query result, comprising an array of feature vectors and the depth level which was actually reached. - * - * @see FeatureArrayIndexer#deepSort(int[]) - * @see FeatureArrayIndexer#deepFill(MaryNode) - */ - public FeatureFileIndexingResult retrieve(FeatureVector v) { - int level = 0; - /* Check if the tree is there */ - if (tree == null) { - throw new RuntimeException("Can't retrieve candidate units if a tree has not been built." - + " (Run this.deepSort(int[]) or this.deepFill(MaryNode) first.)"); - } - /* Walk down the tree */ - MaryNode n = tree; - MaryNode next = null; - while (!n.isLeaf()) { - next = n.getChild(v.getFeatureAsInt(n.getFeatureIndex())); - /* Check if the next node is a dead branch */ - if (next != null) { - n = next; - level++; - } else - break; - } - /* Dereference the reached node or leaf */ - FeatureFileIndexingResult qr = new FeatureFileIndexingResult(getFeatureVectors(n.from, n.to), level); - return (qr); - } - - /** - * Retrieve an array of unit features which complies with a specific target specification, according to an underlying tree, - * and given a stopping condition. - * - * @param v - * A feature vector for which to send back an array of complying unit indexes. - * @param condition - * A constant indicating a stopping criterion, among: FeatureFileIndexer.MAXDEPTH : walk the tree until its leaves - * (maximum depth); FeatureFileIndexer.MAXLEVEL : walk the tree until a certain depth level; - * FeatureFileIndexer.MINUNITS : walk the tree until a certain number of units is reached. - * @param parameter - * A parameter interpreted according to the above condition: MAXDEPTH -> parameter is ignored; MAXLEVEL -> - * parameter = maximum level to reach; MINUNITS -> parameter = lower bound on the number of units to return. - * - * @return A query result, comprising an array of feature vectors and the depth level which was actually reached. - * - * @see FeatureArrayIndexer#deepSort(int[]) - * @see FeatureArrayIndexer#deepFill(MaryNode) - */ - public static final int MAXDEPTH = 0; - public static final int MAXLEVEL = 1; - public static final int MINUNITS = 2; - - public FeatureFileIndexingResult retrieve(FeatureVector v, int condition, int parameter) { - int level = 0; - /* Check if the tree is there */ - if (tree == null) { - throw new RuntimeException("Can't retrieve candidate units if a tree has not been built." - + " (Run this.deepSort(int[]) or this.deepFill(MaryNode) first.)"); - } - // /**/ - // /* TODO: Do we want the warning below? */ - // /*if ( (condition == MAXLEVEL) && (featureSequence != null) && (parameter > featureSequence.length) ) { - // System.out.println( "WARNING: you asked for more levels [" + maxLevel - // + "] than the length of the underlying feature sequence[" + featureSequence.length + "]. Proceeding anyways." ); - // }*/ - /* Walk down the tree */ - MaryNode n = tree; - MaryNode next = null; - while (!n.isLeaf()) { - next = n.getChild(v.getFeatureAsInt(n.getFeatureIndex())); - /* Check for the number of units in the next node */ - if ((condition == MINUNITS) && ((next.to - next.from) < parameter)) - break; - /* Check if the next node is a dead branch */ - if (next != null) { - n = next; - level++; - } else - break; - /* Check for the current level */ - if ((condition == MAXLEVEL) && (level == parameter)) - break; - } - /* Dereference the reached node or leaf */ - FeatureFileIndexingResult qr = new FeatureFileIndexingResult(getFeatureVectors(n.from, n.to), level); - return (qr); - } - - /***************************/ - /* MISCELLANEOUS ACCESSORS */ - /***************************/ - - /** - * Get the feature sequence, as an information about the underlying tree structure. - * - * @return the feature sequence - */ - public int[] getFeatureSequence() { - return featureSequence; - } - - /** - * Get the tree - * - * @return the tree - */ - public MaryNode getTree() { - return tree; - } - - /** - * Get the feature vectors from the big array according to the given indices - * - * @param from - * the start index - * @param to - * the end index - * @return the feature vectors - */ - public FeatureVector[] getFeatureVectors(int from, int to) { - FeatureVector[] vectors = new FeatureVector[to - from]; - for (int i = from; i < to; i++) { - vectors[i - from] = featureVectors[i]; - } - return vectors; - } - - public FeatureDefinition getFeatureDefinition() { - return featureDefinition; - } - - /** - * Get the number of leaves. - * - * @return The number of leaves, or -1 if the tree has not been computed. - */ - public long getNumberOfLeaves() { - if (tree == null) - return (-1); - return (numberOfLeaves); - } - - /** - * Get the theoretical number of leaves, given a feature sequence. - * - * @return The number of leaves, or -1 if the capacity of the long integer was blown. - */ - public long getTheoreticalNumberOfLeaves(int[] feaSeq) { - long ret = 1; - for (int i = 0; i < feaSeq.length; i++) { - // System.out.println( "Feature [" + i + "] has [" + featureDefinition.getNumberOfValues( featureSequence[i] ) + - // "] values." - // + "(Number of leaves = [" + ret + "].)" ); - ret *= featureDefinition.getNumberOfValues(feaSeq[i]); - if (ret < 0) - return (-1); - } - return (ret); - } -} diff --git a/marytts-unitselection/src/main/java/marytts/cart/impose/FeatureComparator.java b/marytts-unitselection/src/main/java/marytts/cart/impose/FeatureComparator.java deleted file mode 100644 index d1b14967f1..0000000000 --- a/marytts-unitselection/src/main/java/marytts/cart/impose/FeatureComparator.java +++ /dev/null @@ -1,118 +0,0 @@ -/** - * Copyright 2006 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.cart.impose; - -import java.util.Comparator; - -import marytts.features.FeatureVector; - -public class FeatureComparator implements Comparator { - - /** The index of the feature to be compared in the feature vector. */ - private int I = -1; - private FeatureVector.FeatureType type = null; - - /** - * Constructor which initializes the feature index. - * - * @param setI - * The index of the feature to be compared on the next run of the comparator. - */ - public FeatureComparator(int setI, FeatureVector.FeatureType featureType) { - setFeatureIdx(setI, featureType); - } - - /** - * Accessor to set the feature index. - * - * @param setI - * The index of the feature to be compared on the next run of the comparator. - */ - public void setFeatureIdx(int setI, FeatureVector.FeatureType featureType) { - I = setI; - type = featureType; - } - - /** - * Access the index of the currently compared feature. - * - * @return The index of the feature which the comparator currently deals with. - */ - public int getFeatureIdx() { - return (I); - } - - /** - * Compares two feature vectors according to their values at an internal index previously set by this.setFeatureIdx(). - * - * @param v1 - * The first vector. - * @param v2 - * The second vector. - * @return a negative integer, zero, or a positive integer as the feature at index I for v1 is less than, equal to, or greater - * than the feature at index I for v2. - * - * @see FeatureComparator#setFeatureIdx(int) - */ - public int compare(FeatureVector a, FeatureVector b) { - switch (type) { - case byteValued: - return a.byteValuedDiscreteFeatures[I] - b.byteValuedDiscreteFeatures[I]; - case shortValued: - int offset = a.byteValuedDiscreteFeatures.length; - return a.shortValuedDiscreteFeatures[I - offset] - b.shortValuedDiscreteFeatures[I - offset]; - case floatValued: - int offset2 = a.byteValuedDiscreteFeatures.length + a.shortValuedDiscreteFeatures.length; - float delta = a.continuousFeatures[I - offset2] - b.continuousFeatures[I - offset2]; - if (delta > 0) - return 1; - else if (delta < 0) - return -1; - return 0; - default: - throw new IllegalStateException("compare called with feature index " + I + " and feature type " + type); - } - - } - - /** - * The equals() method asked for by the Comparable interface. Returns true if the compared object is a FeatureComparator with - * the same internal index, false otherwise. - */ - public boolean equals(Object obj) { - if (!(obj instanceof FeatureComparator)) - return false; - else if (((FeatureComparator) obj).getFeatureIdx() != this.I) - return false; - return (true); - } -} - -/** - * An additional comparator for the unit indexes in the feature vectors. - * - */ -class UnitIndexComparator implements Comparator { - - public int compare(FeatureVector a, FeatureVector b) { - return a.unitIndex - b.unitIndex; - } - -} diff --git a/marytts-unitselection/src/main/java/marytts/cart/impose/FeatureFileIndexingResult.java b/marytts-unitselection/src/main/java/marytts/cart/impose/FeatureFileIndexingResult.java deleted file mode 100644 index bf9e01b157..0000000000 --- a/marytts-unitselection/src/main/java/marytts/cart/impose/FeatureFileIndexingResult.java +++ /dev/null @@ -1,46 +0,0 @@ -/** - * Copyright 2006 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.cart.impose; - -import marytts.features.FeatureVector; - -/** - * A helper class to return the query results from the FeatureFileIndexer. - * - * @author sacha - * - */ -public class FeatureFileIndexingResult { - public FeatureVector[] v = null; - public int level = -1; - - public FeatureFileIndexingResult(FeatureVector[] setV, int setLevel) { - this.v = setV; - this.level = setLevel; - } - - public int[] getUnitIndexes() { - int[] ret = new int[v.length]; - for (int i = 0; i < v.length; i++) { - ret[i] = v[i].getUnitIndex(); - } - return (ret); - } -} diff --git a/marytts-unitselection/src/main/java/marytts/cart/impose/MaryNode.java b/marytts-unitselection/src/main/java/marytts/cart/impose/MaryNode.java deleted file mode 100644 index 708d71e3eb..0000000000 --- a/marytts-unitselection/src/main/java/marytts/cart/impose/MaryNode.java +++ /dev/null @@ -1,133 +0,0 @@ -/** - * Copyright 2006 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.cart.impose; - -import marytts.features.FeatureVector; - -/** - * A generic node class for the tree structures. - * - * @author sacha - * - */ -public class MaryNode { - - protected int featureIndex = -1; - protected int from = 0; - protected int to = 0; - private MaryNode[] kids = null; - - /***************************/ - /* Constructor */ - public MaryNode(int setFrom, int setTo) { - from = setFrom; - to = setTo; - kids = null; - } - - /******************************/ - /* Getters for various fields */ - public int getFrom() { - return (from); - } - - public int getTo() { - return (to); - } - - public int getNumChildren() { - return (kids.length); - } - - public MaryNode[] getChildren() { - return (kids); - } - - /******************************/ - /* Feature index management */ - public void setFeatureIndex(int i) { - featureIndex = i; - } - - public int getFeatureIndex() { - return (featureIndex); - } - - /***************************/ - /* Node splitting */ - public void split(int numKids) { - kids = new MaryNode[numKids]; - } - - public void setChild(int i, MaryNode n) { - kids[i] = n; - } - - public MaryNode getChild(int i) { - return (kids[i]); - } - - /*************************************/ - /* Check if this is a node or a leaf */ - public boolean isNode() { - return (kids != null); - } - - public boolean isLeaf() { - return (kids == null); - } - - // debug output - public void toStandardOut(FeatureArrayIndexer ffi, int level) { - - String blanks = ""; - for (int i = 0; i < level; i++) - blanks += " "; - - if (kids != null) { - String featureName = ffi.getFeatureDefinition().getFeatureName(featureIndex); - System.out.println("Node " + featureName + " has " + (to - from) + " units divided into " + kids.length - + " branches."); - for (int i = 0; i < kids.length; i++) { - if (kids[i] != null) { - System.out.print(blanks + "Branch " + i + "/" + kids.length + " ( " - + ffi.getFeatureDefinition().getFeatureName(featureIndex) + " is " - + ffi.getFeatureDefinition().getFeatureValueAsString(featureIndex, i) + " )" + " -> "); - kids[i].toStandardOut(ffi, level + 1); - } else { - System.out.println(blanks + "Branch " + i + "/" + kids.length + " ( " - + ffi.getFeatureDefinition().getFeatureName(featureIndex) + " is " - + ffi.getFeatureDefinition().getFeatureValueAsString(featureIndex, i) + " )" - + " -> DEAD BRANCH (0 units)"); - } - } - } else { - // get the unit indices - FeatureVector[] fv = ffi.getFeatureVectors(from, to); - System.out.print("LEAF has " + (to - from) + " units : "); - for (int i = 0; i < fv.length; i++) { - System.out.print(fv[i].getUnitIndex() + " "); - } - System.out.print("\n"); - } - - } - -} diff --git a/marytts-unitselection/src/main/java/marytts/cart/io/DirectedGraphReader.java b/marytts-unitselection/src/main/java/marytts/cart/io/DirectedGraphReader.java deleted file mode 100644 index ec57b3f8d8..0000000000 --- a/marytts-unitselection/src/main/java/marytts/cart/io/DirectedGraphReader.java +++ /dev/null @@ -1,290 +0,0 @@ -/** - * Copyright 2006 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.cart.io; - -import java.io.BufferedInputStream; -import java.io.ByteArrayInputStream; -import java.io.DataInput; -import java.io.DataInputStream; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.util.Properties; - -import marytts.cart.DecisionNode; -import marytts.cart.DirectedGraph; -import marytts.cart.DirectedGraphNode; -import marytts.cart.LeafNode; -import marytts.cart.Node; -import marytts.exceptions.MaryConfigurationException; -import marytts.features.FeatureDefinition; -import marytts.util.data.MaryHeader; - -/** - * IO functions for Directed graphs in Mary format - * - * @author Marcela Charfuelan, Marc Schröder - */ -public class DirectedGraphReader { - /** Bit code for identifying a node id as a leaf node id in binary DirectedGraph files */ - public static int LEAFNODE = 0; - /** Bit code for identifying a node id as a decision node id in binary DirectedGraph files */ - public static int DECISIONNODE = 1; - /** Bit code for identifying a node id as a directed node id in binary DirectedGraph files */ - public static int DIRECTEDGRAPHNODE = 2; - - /** - * Load the directed graph from the given file - * - * @param fileName - * the file to load the cart from - * @param featDefinition - * the feature definition - * @param dummy - * unused, just here for compatibility with the FeatureFileIndexer. - * @throws IOException - * , {@link MaryConfigurationException} if a problem occurs while loading - */ - public DirectedGraph load(String fileName) throws IOException, MaryConfigurationException { - InputStream is = new FileInputStream(fileName); - try { - return load(is); - } finally { - is.close(); - } - } - - /** - * Load the directed graph from the given file - * - * @param fileName - * the file to load the cart from - * @param featDefinition - * the feature definition - * @param dummy - * unused, just here for compatibility with the FeatureFileIndexer. - * @throws IOException - * , {@link MaryConfigurationException} if a problem occurs while loading - */ - public DirectedGraph load(InputStream inStream) throws IOException, MaryConfigurationException { - BufferedInputStream buffInStream = new BufferedInputStream(inStream); - assert buffInStream.markSupported(); - buffInStream.mark(10000); - // open the CART-File and read the header - DataInput raf = new DataInputStream(buffInStream); - - MaryHeader maryHeader = new MaryHeader(raf); - if (!maryHeader.hasCurrentVersion()) { - throw new IOException("Wrong version of database file"); - } - if (maryHeader.getType() != MaryHeader.DIRECTED_GRAPH) { - if (maryHeader.getType() == MaryHeader.CARTS) { - buffInStream.reset(); - return new MaryCARTReader().loadFromStream(buffInStream); - } else { - throw new IOException("Not a directed graph file"); - } - } - - // Read properties - short propDataLength = raf.readShort(); - Properties props; - if (propDataLength == 0) { - props = null; - } else { - byte[] propsData = new byte[propDataLength]; - raf.readFully(propsData); - ByteArrayInputStream bais = new ByteArrayInputStream(propsData); - props = new Properties(); - props.load(bais); - bais.close(); - } - - // Read the feature definition - FeatureDefinition featureDefinition = new FeatureDefinition(raf); - - // read the decision nodes - int numDecNodes = raf.readInt(); // number of decision nodes - - // First we need to read all nodes into memory, then we can link them properly - // in terms of parent/child. - DecisionNode[] dns = new DecisionNode[numDecNodes]; - int[][] childIndexes = new int[numDecNodes][]; - for (int i = 0; i < numDecNodes; i++) { - // read one decision node - int featureNameIndex = raf.readInt(); - int nodeTypeNr = raf.readInt(); - DecisionNode.Type nodeType = DecisionNode.Type.values()[nodeTypeNr]; - int numChildren = 2; // for binary nodes - switch (nodeType) { - case BinaryByteDecisionNode: - int criterion = raf.readInt(); - dns[i] = new DecisionNode.BinaryByteDecisionNode(featureNameIndex, (byte) criterion, featureDefinition); - break; - case BinaryShortDecisionNode: - criterion = raf.readInt(); - dns[i] = new DecisionNode.BinaryShortDecisionNode(featureNameIndex, (short) criterion, featureDefinition); - break; - case BinaryFloatDecisionNode: - float floatCriterion = raf.readFloat(); - dns[i] = new DecisionNode.BinaryFloatDecisionNode(featureNameIndex, floatCriterion, featureDefinition); - break; - case ByteDecisionNode: - numChildren = raf.readInt(); - if (featureDefinition.getNumberOfValues(featureNameIndex) != numChildren) { - throw new IOException("Inconsistent cart file: feature " + featureDefinition.getFeatureName(featureNameIndex) - + " should have " + featureDefinition.getNumberOfValues(featureNameIndex) - + " values, but decision node " + i + " has only " + numChildren + " child nodes"); - } - dns[i] = new DecisionNode.ByteDecisionNode(featureNameIndex, numChildren, featureDefinition); - break; - case ShortDecisionNode: - numChildren = raf.readInt(); - if (featureDefinition.getNumberOfValues(featureNameIndex) != numChildren) { - throw new IOException("Inconsistent cart file: feature " + featureDefinition.getFeatureName(featureNameIndex) - + " should have " + featureDefinition.getNumberOfValues(featureNameIndex) - + " values, but decision node " + i + " has only " + numChildren + " child nodes"); - } - dns[i] = new DecisionNode.ShortDecisionNode(featureNameIndex, numChildren, featureDefinition); - } - dns[i].setUniqueDecisionNodeId(i + 1); - // now read the children, indexes only: - childIndexes[i] = new int[numChildren]; - for (int k = 0; k < numChildren; k++) { - childIndexes[i][k] = raf.readInt(); - } - } - - // read the leaves - int numLeafNodes = raf.readInt(); // number of leaves, it does not include empty leaves - LeafNode[] lns = new LeafNode[numLeafNodes]; - - for (int j = 0; j < numLeafNodes; j++) { - // read one leaf node - int leafTypeNr = raf.readInt(); - LeafNode.LeafType leafNodeType = LeafNode.LeafType.values()[leafTypeNr]; - switch (leafNodeType) { - case IntArrayLeafNode: - int numData = raf.readInt(); - int[] data = new int[numData]; - for (int d = 0; d < numData; d++) { - data[d] = raf.readInt(); - } - lns[j] = new LeafNode.IntArrayLeafNode(data); - break; - case FloatLeafNode: - float stddev = raf.readFloat(); - float mean = raf.readFloat(); - lns[j] = new LeafNode.FloatLeafNode(new float[] { stddev, mean }); - break; - case IntAndFloatArrayLeafNode: - case StringAndFloatLeafNode: - int numPairs = raf.readInt(); - int[] ints = new int[numPairs]; - float[] floats = new float[numPairs]; - for (int d = 0; d < numPairs; d++) { - ints[d] = raf.readInt(); - floats[d] = raf.readFloat(); - } - if (leafNodeType == LeafNode.LeafType.IntAndFloatArrayLeafNode) - lns[j] = new LeafNode.IntAndFloatArrayLeafNode(ints, floats); - else - lns[j] = new LeafNode.StringAndFloatLeafNode(ints, floats); - break; - case FeatureVectorLeafNode: - throw new IllegalArgumentException("Reading feature vector leaf nodes is not yet implemented"); - case PdfLeafNode: - throw new IllegalArgumentException("Reading pdf leaf nodes is not yet implemented"); - } - lns[j].setUniqueLeafId(j + 1); - } - - // Graph nodes - int numDirectedGraphNodes = raf.readInt(); - DirectedGraphNode[] graphNodes = new DirectedGraphNode[numDirectedGraphNodes]; - int[] dgnLeafIndices = new int[numDirectedGraphNodes]; - int[] dgnDecIndices = new int[numDirectedGraphNodes]; - for (int g = 0; g < numDirectedGraphNodes; g++) { - graphNodes[g] = new DirectedGraphNode(null, null); - graphNodes[g].setUniqueGraphNodeID(g + 1); - dgnLeafIndices[g] = raf.readInt(); - dgnDecIndices[g] = raf.readInt(); - } - - // Now, link up the decision nodes with their daughters - for (int i = 0; i < numDecNodes; i++) { - // System.out.print(dns[i]+" "+dns[i].getFeatureName()+" "); - for (int k = 0; k < childIndexes[i].length; k++) { - Node child = childIndexToNode(childIndexes[i][k], dns, lns, graphNodes); - dns[i].addDaughter(child); - // System.out.print(" "+dns[i].getDaughter(k)); - } - // System.out.println(); - } - // And link up directed graph nodes - for (int g = 0; g < numDirectedGraphNodes; g++) { - Node leaf = childIndexToNode(dgnLeafIndices[g], dns, lns, graphNodes); - graphNodes[g].setLeafNode(leaf); - Node dec = childIndexToNode(dgnDecIndices[g], dns, lns, graphNodes); - if (dec != null && !dec.isDecisionNode()) - throw new IllegalArgumentException("Only decision nodes allowed, read " + dec.getClass()); - graphNodes[g].setDecisionNode((DecisionNode) dec); - // System.out.println("Graph node "+(g+1)+", leaf: "+Integer.toHexString(dgnLeafIndices[g])+", "+leaf+" -- dec: "+Integer.toHexString(dgnDecIndices[g])+", "+dec); - } - - Node rootNode; - if (graphNodes.length > 0) { - rootNode = graphNodes[0]; - } else if (dns.length > 0) { - rootNode = dns[0]; - // CART behaviour, not sure if this is needed: - // Now count all data once, so that getNumberOfData() - // will return the correct figure. - ((DecisionNode) rootNode).countData(); - } else if (lns.length > 0) { - rootNode = lns[0]; // single-leaf tree... - } else { - rootNode = null; - } - - // set the rootNode as the rootNode of cart - return new DirectedGraph(rootNode, featureDefinition, props); - } - - private Node childIndexToNode(int childIndexAndType, DecisionNode[] dns, LeafNode[] lns, DirectedGraphNode[] graphNodes) { - int childIndex = childIndexAndType & 0x3fffffff; // the lower 30 bits - int childType = (childIndexAndType >> 30) & 0x03; // the highest two bits - if (childIndex == 0) { // an empty leaf - return null; - } else if (childType == DECISIONNODE) { // a decision node - assert childIndex - 1 < dns.length; - return dns[childIndex - 1]; - } else if (childType == LEAFNODE) { // a leaf node - assert childIndex - 1 < lns.length; - return lns[childIndex - 1]; - } else if (childType == DIRECTEDGRAPHNODE) { - assert childIndex - 1 < graphNodes.length; - return graphNodes[childIndex - 1]; - } else { - throw new IllegalArgumentException("Unexpected child type: " + childType); - } - - } -} diff --git a/marytts-unitselection/src/main/java/marytts/cart/io/DirectedGraphWriter.java b/marytts-unitselection/src/main/java/marytts/cart/io/DirectedGraphWriter.java deleted file mode 100644 index 215d254098..0000000000 --- a/marytts-unitselection/src/main/java/marytts/cart/io/DirectedGraphWriter.java +++ /dev/null @@ -1,409 +0,0 @@ -/** - * Copyright 2006 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.cart.io; - -import java.io.BufferedOutputStream; -import java.io.ByteArrayOutputStream; -import java.io.DataOutput; -import java.io.DataOutputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.PrintWriter; -import java.util.Properties; - -import marytts.cart.DecisionNode; -import marytts.cart.DirectedGraph; -import marytts.cart.DirectedGraphNode; -import marytts.cart.LeafNode; -import marytts.cart.Node; -import marytts.cart.DecisionNode.BinaryByteDecisionNode; -import marytts.cart.DecisionNode.BinaryFloatDecisionNode; -import marytts.cart.DecisionNode.BinaryShortDecisionNode; -import marytts.cart.LeafNode.FeatureVectorLeafNode; -import marytts.cart.LeafNode.FloatLeafNode; -import marytts.cart.LeafNode.IntAndFloatArrayLeafNode; -import marytts.cart.LeafNode.IntArrayLeafNode; -import marytts.cart.LeafNode.LeafType; -import marytts.features.FeatureVector; -import marytts.util.MaryUtils; -import marytts.util.data.MaryHeader; - -import org.apache.log4j.Logger; - -/** - * IO functions for directed graphs in Mary format - * - * @author Marcela Charfuelan, Marc Schröder - */ -public class DirectedGraphWriter { - - protected Logger logger = MaryUtils.getLogger(this.getClass().getName()); - - /** - * Dump the graph in Mary format - * - * @param destDir - * the destination directory - */ - public void saveGraph(DirectedGraph graph, String destFile) throws IOException { - if (graph == null) - throw new NullPointerException("Cannot dump null graph"); - if (destFile == null) - throw new NullPointerException("No destination file"); - - logger.debug("Dumping directed graph in Mary format to " + destFile + " ..."); - - // Open the destination file and output the header - DataOutputStream out = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(destFile))); - // create new CART-header and write it to output file - MaryHeader hdr = new MaryHeader(MaryHeader.DIRECTED_GRAPH); - hdr.writeTo(out); - - Properties props = graph.getProperties(); - if (props == null) { - out.writeShort(0); - } else { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - props.store(baos, null); - byte[] propData = baos.toByteArray(); - out.writeShort(propData.length); - out.write(propData); - } - - // feature definition - graph.getFeatureDefinition().writeBinaryTo(out); - - // dump graph - dumpBinary(graph, out); - - // finish - out.close(); - logger.debug(" ... done\n"); - } - - public void toTextOut(DirectedGraph graph, PrintWriter pw) throws IOException { - try { - int numLeafNodes = setUniqueLeafNodeIds(graph); - int numDecNodes = setUniqueDecisionNodeIds(graph); - int numGraphNodes = setUniqueDirectedGraphNodeIds(graph); - pw.println("Num decision nodes= " + numDecNodes + " Num leaf nodes= " + numLeafNodes - + " Num directed graph nodes= " + numGraphNodes); - printDecisionNodes(graph, null, pw); - pw.println("\n----------------\n"); - printLeafNodes(graph, null, pw); - pw.println("\n----------------\n"); - printDirectedGraphNodes(graph, null, pw); - - pw.flush(); - pw.close(); - } catch (IOException ioe) { - IOException newIOE = new IOException("Error dumping graph to standard output"); - newIOE.initCause(ioe); - throw newIOE; - } - } - - /** - * Assign unique ids to leaf nodes. - * - * @param graph - * @return the number of different leaf nodes - */ - private int setUniqueLeafNodeIds(DirectedGraph graph) { - int i = 0; - for (LeafNode l : graph.getLeafNodes()) { - l.setUniqueLeafId(++i); - } - return i; - } - - /** - * Assign unique ids to decision nodes. - * - * @param graph - * @return the number of different decision nodes - */ - private int setUniqueDecisionNodeIds(DirectedGraph graph) { - int i = 0; - for (DecisionNode d : graph.getDecisionNodes()) { - d.setUniqueDecisionNodeId(++i); - } - return i; - } - - /** - * Assign unique ids to directed graph nodes. - * - * @param graph - * @return the number of different directed graph nodes - */ - private int setUniqueDirectedGraphNodeIds(DirectedGraph graph) { - int i = 0; - for (DirectedGraphNode g : graph.getDirectedGraphNodes()) { - g.setUniqueGraphNodeID(++i); - } - return i; - } - - private void dumpBinary(DirectedGraph graph, DataOutput os) throws IOException { - try { - int numLeafNodes = setUniqueLeafNodeIds(graph); - int numDecNodes = setUniqueDecisionNodeIds(graph); - int numGraphNodes = setUniqueDirectedGraphNodeIds(graph); - int maxNum = 1 << 30; - if (numLeafNodes > maxNum || numDecNodes > maxNum || numGraphNodes > maxNum) { - throw new UnsupportedOperationException("Cannot write more than " + maxNum + " nodes of one type in this format"); - } - // write the number of decision nodes - os.writeInt(numDecNodes); - printDecisionNodes(graph, os, null); - - // write the number of leaves. - os.writeInt(numLeafNodes); - printLeafNodes(graph, os, null); - - // write the number of directed graph nodes - os.writeInt(numGraphNodes); - printDirectedGraphNodes(graph, os, null); - - } catch (IOException ioe) { - IOException newIOE = new IOException("Error dumping CART to output stream"); - newIOE.initCause(ioe); - throw newIOE; - } - } - - private void printDecisionNodes(DirectedGraph graph, DataOutput out, PrintWriter pw) throws IOException { - for (DecisionNode decNode : graph.getDecisionNodes()) { - int id = decNode.getUniqueDecisionNodeId(); - String nodeDefinition = decNode.getNodeDefinition(); - int featureIndex = decNode.getFeatureIndex(); - DecisionNode.Type nodeType = decNode.getDecisionNodeType(); - - if (out != null) { - // dump in binary form to output - out.writeInt(featureIndex); - out.writeInt(nodeType.ordinal()); - // Now, questionValue, which depends on nodeType - switch (nodeType) { - case BinaryByteDecisionNode: - out.writeInt(((BinaryByteDecisionNode) decNode).getCriterionValueAsByte()); - assert decNode.getNumberOfDaugthers() == 2; - break; - case BinaryShortDecisionNode: - out.writeInt(((BinaryShortDecisionNode) decNode).getCriterionValueAsShort()); - assert decNode.getNumberOfDaugthers() == 2; - break; - case BinaryFloatDecisionNode: - out.writeFloat(((BinaryFloatDecisionNode) decNode).getCriterionValueAsFloat()); - assert decNode.getNumberOfDaugthers() == 2; - break; - case ByteDecisionNode: - case ShortDecisionNode: - out.writeInt(decNode.getNumberOfDaugthers()); - } - - // The child nodes - for (int i = 0, n = decNode.getNumberOfDaugthers(); i < n; i++) { - Node daughter = decNode.getDaughter(i); - if (daughter == null) { - out.writeInt(0); - } else if (daughter.isDecisionNode()) { - int daughterID = ((DecisionNode) daughter).getUniqueDecisionNodeId(); - // Mark as decision node: - daughterID |= DirectedGraphReader.DECISIONNODE << 30; - out.writeInt(daughterID); - } else if (daughter.isLeafNode()) { - int daughterID = ((LeafNode) daughter).getUniqueLeafId(); - // Mark as leaf node: - if (daughterID != 0) - daughterID |= DirectedGraphReader.LEAFNODE << 30; - out.writeInt(daughterID); - } else if (daughter.isDirectedGraphNode()) { - int daughterID = ((DirectedGraphNode) daughter).getUniqueGraphNodeID(); - // Mark as directed graph node: - if (daughterID != 0) - daughterID |= DirectedGraphReader.DIRECTEDGRAPHNODE << 30; - out.writeInt(daughterID); - } - } - } - if (pw != null) { - // dump to print writer - StringBuilder strNode = new StringBuilder("-" + id + " " + nodeDefinition); - for (int i = 0, n = decNode.getNumberOfDaugthers(); i < n; i++) { - strNode.append(" "); - Node daughter = decNode.getDaughter(i); - if (daughter == null) { - strNode.append("0"); - } else if (daughter.isDecisionNode()) { - int daughterID = ((DecisionNode) daughter).getUniqueDecisionNodeId(); - strNode.append("-").append(daughterID); - out.writeInt(daughterID); - } else if (daughter.isLeafNode()) { - int daughterID = ((LeafNode) daughter).getUniqueLeafId(); - if (daughterID == 0) - strNode.append("0"); - else - strNode.append("id").append(daughterID); - } else if (daughter.isDirectedGraphNode()) { - int daughterID = ((DirectedGraphNode) daughter).getUniqueGraphNodeID(); - if (daughterID == 0) - strNode.append("0"); - else - strNode.append("DGN").append(daughterID); - } - } - pw.println(strNode.toString()); - } - } - } - - private void printLeafNodes(DirectedGraph graph, DataOutput out, PrintWriter pw) throws IOException { - for (LeafNode leaf : graph.getLeafNodes()) { - if (leaf.getUniqueLeafId() == 0) // empty leaf, do not write - continue; - LeafType leafType = leaf.getLeafNodeType(); - if (leafType == LeafType.FeatureVectorLeafNode) { - leafType = LeafType.IntArrayLeafNode; - // save feature vector leaf nodes as int array leaf nodes - } - if (out != null) { - // Leaf node type - out.writeInt(leafType.ordinal()); - } - if (pw != null) { - pw.print("id" + leaf.getUniqueLeafId() + " " + leafType); - } - switch (leaf.getLeafNodeType()) { - case IntArrayLeafNode: - int data[] = ((IntArrayLeafNode) leaf).getIntData(); - // Number of data points following: - if (out != null) - out.writeInt(data.length); - if (pw != null) - pw.print(" " + data.length); - // for each index, write the index - for (int i = 0; i < data.length; i++) { - if (out != null) - out.writeInt(data[i]); - if (pw != null) - pw.print(" " + data[i]); - } - break; - case FloatLeafNode: - float stddev = ((FloatLeafNode) leaf).getStDeviation(); - float mean = ((FloatLeafNode) leaf).getMean(); - if (out != null) { - out.writeFloat(stddev); - out.writeFloat(mean); - } - if (pw != null) { - pw.print(" 1 " + stddev + " " + mean); - } - break; - case IntAndFloatArrayLeafNode: - case StringAndFloatLeafNode: - int data1[] = ((IntAndFloatArrayLeafNode) leaf).getIntData(); - float floats[] = ((IntAndFloatArrayLeafNode) leaf).getFloatData(); - // Number of data points following: - if (out != null) - out.writeInt(data1.length); - if (pw != null) - pw.print(" " + data1.length); - // for each index, write the index and then its float - for (int i = 0; i < data1.length; i++) { - if (out != null) { - out.writeInt(data1[i]); - out.writeFloat(floats[i]); - } - if (pw != null) - pw.print(" " + data1[i] + " " + floats[i]); - } - break; - case FeatureVectorLeafNode: - FeatureVector fv[] = ((FeatureVectorLeafNode) leaf).getFeatureVectors(); - // Number of data points following: - if (out != null) - out.writeInt(fv.length); - if (pw != null) - pw.print(" " + fv.length); - // for each feature vector, write the index - for (int i = 0; i < fv.length; i++) { - if (out != null) - out.writeInt(fv[i].getUnitIndex()); - if (pw != null) - pw.print(" " + fv[i].getUnitIndex()); - } - break; - case PdfLeafNode: - throw new IllegalArgumentException("Writing of pdf leaf nodes not yet implemented"); - } - if (pw != null) - pw.println(); - } - } - - private void printDirectedGraphNodes(DirectedGraph graph, DataOutput out, PrintWriter pw) throws IOException { - for (DirectedGraphNode g : graph.getDirectedGraphNodes()) { - int id = g.getUniqueGraphNodeID(); - if (id == 0) - continue;// empty node, do not write - Node leaf = g.getLeafNode(); - int leafID = 0; - int leafNodeType = 0; - if (leaf != null) { - if (leaf instanceof LeafNode) { - leafID = ((LeafNode) leaf).getUniqueLeafId(); - leafNodeType = DirectedGraphReader.LEAFNODE; - } else if (leaf instanceof DirectedGraphNode) { - leafID = ((DirectedGraphNode) leaf).getUniqueGraphNodeID(); - leafNodeType = DirectedGraphReader.DIRECTEDGRAPHNODE; - } else { - throw new IllegalArgumentException("Unexpected leaf type: " + leaf.getClass()); - } - } - DecisionNode d = g.getDecisionNode(); - int decID = d != null ? d.getUniqueDecisionNodeId() : 0; - if (out != null) { - int outLeafId = leafID == 0 ? 0 : leafID | (leafNodeType << 30); - out.writeInt(outLeafId); - int outDecId = decID == 0 ? 0 : decID | (DirectedGraphReader.DECISIONNODE << 30); - out.writeInt(outDecId); - } - if (pw != null) { - pw.print("DGN" + id); - if (leafID == 0) { - pw.print(" 0"); - } else if (leaf.isLeafNode()) { - pw.print(" id" + leafID); - } else { - assert leaf.isDirectedGraphNode(); - pw.print(" DGN" + leafID); - } - if (decID == 0) - pw.print(" 0"); - else - pw.print(" -" + decID); - pw.println(); - } - } - } -} diff --git a/marytts-unitselection/src/main/java/marytts/cart/io/HTSCARTReader.java b/marytts-unitselection/src/main/java/marytts/cart/io/HTSCARTReader.java deleted file mode 100644 index fba530ebfc..0000000000 --- a/marytts-unitselection/src/main/java/marytts/cart/io/HTSCARTReader.java +++ /dev/null @@ -1,556 +0,0 @@ -/** - * The HMM-Based Speech Synthesis System (HTS) - * HTS Working Group - * - * Department of Computer Science - * Nagoya Institute of Technology - * and - * Interdisciplinary Graduate School of Science and Engineering - * Tokyo Institute of Technology - * - * Portions Copyright (c) 2001-2006 - * All Rights Reserved. - * - * Portions Copyright 2000-2007 DFKI GmbH. - * All Rights Reserved. - * - * Permission is hereby granted, free of charge, to use and - * distribute this software and its documentation without - * restriction, including without limitation the rights to use, - * copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of this work, and to permit persons to whom this - * work is furnished to do so, subject to the following conditions: - * - * 1. The source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * - * 2. Any modifications to the source code must be clearly - * marked as such. - * - * 3. Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the - * following disclaimer in the documentation and/or other - * materials provided with the distribution. Otherwise, one - * must contact the HTS working group. - * - * NAGOYA INSTITUTE OF TECHNOLOGY, TOKYO INSTITUTE OF TECHNOLOGY, - * HTS WORKING GROUP, AND THE CONTRIBUTORS TO THIS WORK DISCLAIM - * ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING ALL - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO EVENT - * SHALL NAGOYA INSTITUTE OF TECHNOLOGY, TOKYO INSTITUTE OF - * TECHNOLOGY, HTS WORKING GROUP, NOR THE CONTRIBUTORS BE LIABLE - * FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY - * DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, - * WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTUOUS - * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR - * PERFORMANCE OF THIS SOFTWARE. - * - */ -package marytts.cart.io; - -import java.io.BufferedInputStream; -import java.io.BufferedReader; -import java.io.DataInputStream; -import java.io.FileInputStream; -import java.io.FileReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.StringReader; -import java.util.Scanner; -import java.util.StringTokenizer; - -import marytts.cart.CART; -import marytts.cart.DecisionNode; -import marytts.cart.LeafNode; -import marytts.cart.Node; -import marytts.cart.DecisionNode.BinaryByteDecisionNode; -import marytts.cart.LeafNode.PdfLeafNode; -import marytts.exceptions.MaryConfigurationException; -import marytts.features.FeatureDefinition; -import marytts.htsengine.PhoneTranslator; -import marytts.htsengine.HMMData.PdfFileFormat; -import marytts.util.MaryUtils; - -import org.apache.log4j.Logger; - -/** - * Reader functions for CARTs in HTS format - * - * @author Marcela Charfuelan - */ -public class HTSCARTReader { - - private FeatureDefinition featDef; - private PhoneTranslator phTrans; - private Logger logger = MaryUtils.getLogger("HTSCARTReader"); - private int vectorSize; // the vector size of the mean and variance on the leaves of the tree. - - public int getVectorSize() { - return vectorSize; - } - - /** - * Load the cart from the given file - * - * @param nummStates - * number of states in the HTS model, it will create one cart tree per state. - * @param treefileName - * the HTS tree text file, example tree-mgc.inf. - * @param pdfFileName - * the corresponding HTS pdf binary file, example mgc.pdf. - * @param featDefinition - * the feature definition - * @param CART - * [] Fills this array of CART trees, one per state. - * @return the size of the mean and variance vectors on the leaves. - * @throws IOException - * if a problem occurs while loading - */ - public CART[] load(int numStates, InputStream treeStream, InputStream pdfStream, PdfFileFormat fileFormat, - FeatureDefinition featDefinition, PhoneTranslator phTranslator) throws IOException, MaryConfigurationException { - - featDef = featDefinition; - // phTrans = phoneTranslator; - int i, j, length, state; - BufferedReader s = null; - String line, aux; - - phTrans = phTranslator; - - // create the number of carts it is going to read - CART treeSet[] = new CART[numStates]; - for (i = 0; i < numStates; i++) - treeSet[i] = new CART(); - - // First load pdfs, so when creates the tree fill the leaf nodes with - // the corresponding mean and variances. - /** - * load pdf's, mean and variance pdfs format : pdf[numStates][numPdfs][numStreams][2*vectorSize] - * ------------------------------------------------------------------- for dur : pdf[ 1 ][numPdfs][ 1 ][2*numStates ] for - * mgc,str,mag: pdf[numStates][numPdfs][ 1 ][2*vectorSize]; for joinModel : pdf[ 1 ][numPdfs][ 1 ][2*vectorSize]; for lf0 - * : pdf[numStates][numPdfs][numStreams][ 4 ] for gv-switch : pdf[ 1 ][ 1 ][ 1 ][ 1 ] - * ------------------------------------------------------------------ - numPdf : corresponds to the unique leaf node id. - - * 2*vectorSize : means that mean and variance are in the same vector. - 4 in lf0 : means 0: mean, 1: variance, 2: voiced - * weight and 3: unvoiced weight ------------------------------------------------------------------ - */ - double pdf[][][][]; - pdf = loadPdfs(numStates, pdfStream, fileFormat); - - assert featDefinition != null : "Feature Definition was not set"; - - /* read lines of tree-*.inf fileName */ - s = new BufferedReader(new InputStreamReader(treeStream, "UTF-8")); - - // skip questions section - while ((line = s.readLine()) != null) { - if (line.indexOf("QS") < 0) - break; /* a new state is indicated by {*}[2], {*}[3], ... */ - } - - while ((line = s.readLine()) != null) { - if (line.indexOf("{*}") >= 0) { /* this is the indicator of a new state-tree */ - aux = line.substring(line.indexOf("[") + 1, line.indexOf("]")); - state = Integer.parseInt(aux); - // loads one cart tree per state - treeSet[state - 2].setRootNode(loadStateTree(s, pdf[state - 2])); - - // Now count all data once, so that getNumberOfData() - // will return the correct figure. - if (treeSet[state - 2].getRootNode() instanceof DecisionNode) - ((DecisionNode) treeSet[state - 2].getRootNode()).countData(); - - logger.debug("load: CART[" + (state - 2) + "], total number of nodes in this CART: " - + treeSet[state - 2].getNumNodes()); - } - } /* while */ - if (s != null) - s.close(); - - /* check that the tree was correctly loaded */ - if (treeSet.length == 0) { - throw new IOException("LoadTreeSet: error no trees loaded"); - } - - return treeSet; - - } - - /** - * Load a tree per state - * - * @param s - * : text scanner of the whole tree-*.inf file - * @param pdf - * : the pdfs for this state, pdf[numPdfs][numStreams][2*vectorSize] - */ - private Node loadStateTree(BufferedReader s, double pdf[][][]) throws IOException, MaryConfigurationException { - - Node rootNode = null; - Node lastNode = null; - - StringTokenizer sline; - String aux, buf; - - // create an empty binary decision node with unique id=0, this will be the rootNode - Node nextNode = new DecisionNode.BinaryByteDecisionNode(0, featDef); - - // this is the rootNode - rootNode = nextNode; - nextNode.setIsRoot(true); - - int iaux, feaIndex, ndec, nleaf; - ndec = 0; - nleaf = 0; - Node node = null; - aux = s.readLine(); /* next line for this state tree must be { */ - int id; - - if (aux.indexOf("{") >= 0) { - while ((aux = s.readLine()) != null && aux.indexOf("}") < 0) { /* last line for this state tree must be } */ - /* then parse this line, it contains 4 fields */ - /* 1: node index # 2: Question name 3: NO # node 4: YES # node */ - sline = new StringTokenizer(aux); - - /* 1: gets index node and looks for the node whose idx = buf */ - buf = sline.nextToken(); - if (buf.startsWith("-")) { - id = Integer.parseInt(buf.substring(1)); - ndec++; - } else if (buf.contentEquals("0")) - id = 0; - else - throw new MaryConfigurationException("LoadStateTree: line does not start with a decision node (-id), line=" - + aux); - // 1. find the node in the tree, it has to be already created. - node = findDecisionNode(rootNode, id); - - if (node == null) - throw new MaryConfigurationException("LoadStateTree: Node not found, index = " + buf); - else { - /* 2: gets question name and question name val */ - buf = sline.nextToken(); - String[] fea_val = buf.split("="); /* splits featureName=featureValue */ - feaIndex = featDef.getFeatureIndex(fea_val[0]); - - /* Replace back punctuation values */ - /* what about tricky phones, if using halfphones it would not be necessary */ - if (fea_val[0].contentEquals("sentence_punc") || fea_val[0].contentEquals("prev_punctuation") - || fea_val[0].contentEquals("next_punctuation")) { - // System.out.print("CART replace punc: " + fea_val[0] + " = " + fea_val[1]); - fea_val[1] = phTrans.replaceBackPunc(fea_val[1]); - // System.out.println(" --> " + fea_val[0] + " = " + fea_val[1]); - } else if (fea_val[0].contains("tobi_")) { - // System.out.print("CART replace tobi: " + fea_val[0] + " = " + fea_val[1]); - fea_val[1] = phTrans.replaceBackToBI(fea_val[1]); - // System.out.println(" --> " + fea_val[0] + " = " + fea_val[1]); - } else if (fea_val[0].contains("phone")) { - // System.out.print("CART replace phone: " + fea_val[0] + " = " + fea_val[1]); - fea_val[1] = phTrans.replaceBackTrickyPhones(fea_val[1]); - // System.out.println(" --> " + fea_val[0] + " = " + fea_val[1]); - } - - // add featureName and featureValue to the decision nod - ((BinaryByteDecisionNode) node).setFeatureAndFeatureValue(fea_val[0], fea_val[1]); - - // add NO and YES indexes to the daughther nodes - /* NO index */ - buf = sline.nextToken(); - if (buf.startsWith("-")) { // Decision node - iaux = Integer.parseInt(buf.substring(1)); - // create an empty binary decision node with unique id - BinaryByteDecisionNode auxnode = new DecisionNode.BinaryByteDecisionNode(iaux, featDef); - ((DecisionNode) node).replaceDaughter(auxnode, 1); - } else { // LeafNode - iaux = Integer.parseInt(buf.substring(buf.lastIndexOf("_") + 1, buf.length() - 1)); - // create an empty PdfLeafNode - PdfLeafNode auxnode = new LeafNode.PdfLeafNode(iaux, pdf[iaux - 1]); - ((DecisionNode) node).replaceDaughter(auxnode, 1); - nleaf++; - } - - /* YES index */ - buf = sline.nextToken(); - if (buf.startsWith("-")) { // Decision node - iaux = Integer.parseInt(buf.substring(1)); - // create an empty binary decision node with unique id=0 - BinaryByteDecisionNode auxnode = new DecisionNode.BinaryByteDecisionNode(iaux, featDef); - ((DecisionNode) node).replaceDaughter(auxnode, 0); - } else { // LeafNode - iaux = Integer.parseInt(buf.substring(buf.lastIndexOf("_") + 1, buf.length() - 1)); - // create an empty PdfLeafNode - PdfLeafNode auxnode = new LeafNode.PdfLeafNode(iaux, pdf[iaux - 1]); - ((DecisionNode) node).replaceDaughter(auxnode, 0); - nleaf++; - } - } /* if node not null */ - sline = null; - } /* while there is another line and the line does not contain } */ - } /* if not "{" */ - - logger.debug("loadStateTree: loaded CART contains " + (ndec + 1) + " Decision nodes and " + nleaf + " Leaf nodes."); - return rootNode; - - } /* method loadTree() */ - - /** - * @param node - * , decision node - * @param numId - * , index to look for. - * @return - */ - private Node findDecisionNode(Node node, int numId) { - - Node aux = null; - - if (node instanceof DecisionNode) { - // System.out.print(" id=" + ((DecisionNode)node).getUniqueDecisionNodeId()); - if (((DecisionNode) node).getUniqueDecisionNodeId() == numId) - return node; - else { - for (int i = 0; i < ((DecisionNode) node).getNumberOfDaugthers(); i++) { - aux = findDecisionNode(((DecisionNode) node).getDaughter(i), numId); - if (aux != null) - return aux; - } - } - } - return aux; - - } /* method findDecisionNode */ - - /** - * Load pdf's, mean and variance the #leaves corresponds to the unique leaf node id pdf --> - * [#states][#leaves][#streams][vectorsize] The format of pdf files for mgc, str or mag is: header: 4 byte int: dimension - * feature vector 4 byte int: # of leaf nodes for state 1 4 byte int: # of leaf nodes for state 2 ... 4 byte int: # of leaf - * nodes for state N probability distributions: 4 byte float means and variances (2*pdfVsize): all leaves for state 1 4 byte - * float means and variances (2*pdfVsize): all leaves for state 2 ... 4 byte float means and variances (2*pdfVsize): all - * leaves for state N --------------------------------------------------------------------- The format of pdf files for dur - * and JoinModeller is: header: 4 byte int: # of HMM states <-- this is the dimension of vector in duration 4 byte int: # of - * leaf nodes for state 1 <-- dur has just one state probability distributions: 4 byte float means and variances (2*HMMsize): - * all leaves for state 1 --------------------------------------------------------------------- The format of pdf files for - * lf0 is: header: 4 byte int: dimension feature vector 4 byte int: # of leaf nodes for state 1 4 byte int: # of leaf nodes - * for state 2 ... 4 byte int: # of leaf nodes for state N probability distributions: 4 byte float mean, variance, voiced, - * unvoiced (4 floats): stream 1..S, leaf 1..L, state 1 4 byte float mean, variance, voiced, unvoiced (4 floats): stream 1..S, - * leaf 1..L, state 2 ... 4 byte float mean, variance, voiced, unvoiced (4 floats): stream 1..S, leaf 1..L, state N - */ - private double[][][][] loadPdfs(int numState, InputStream pdfStream, PdfFileFormat fileFormat) throws IOException, - MaryConfigurationException { - - DataInputStream data_in; - int i, j, k, l, numDurPdf, lf0Stream; - double vw, uvw; - int vsize; - int numPdf[]; - int numStream; - int numMSDFlag; /* MSD: Multi stream dimensions: in case of lf0 for example */ - double pdf[][][][] = null; // pdf[numState][numPdf][stream][vsize]; - - // TODO: how to make this loading more general, different files have different formats. Right now the way - // of loading depends on the name of the file, I need to change that! - // pdfFileName.contains("dur.pdf") || pdfFileName.contains("joinModeller.pdf") - if (fileFormat == PdfFileFormat.dur || fileFormat == PdfFileFormat.join) { - /* ________________________________________________________________ */ - /*-------------------- load pdfs for duration --------------------*/ - data_in = new DataInputStream(new BufferedInputStream(pdfStream)); - logger.debug("loadPdfs reading model of type " + fileFormat); - - /* read the number of states & the number of pdfs (leaf nodes) */ - /* read the number of HMM states, this number is the same for all pdf's. */ - - numMSDFlag = data_in.readInt(); - numStream = data_in.readInt(); - vectorSize = data_in.readInt(); - // ---vectorSize = numState; - // System.out.println("loadPdfs: nstate = " + nstate); - - numState = numStream; - - /* check number of states */ - if (numState < 0) - throw new MaryConfigurationException("loadPdfs: #HMM states must be positive value."); - - /* read the number of duration pdfs */ - numDurPdf = data_in.readInt(); - logger.debug("loadPdfs: numPdf[state:0]=" + numDurPdf); - - /* Now we know the number of duration pdfs and the vector size which is */ - /* the number of states in each HMM. Here the vector size is 2*nstate because */ - /* the first nstate correspond to the mean and the second nstate correspond */ - /* to the diagonal variance vector, the mean and variance are copied here in */ - /* only one vector. */ - /* 2*nstate because the vector size for duration is the number of states */ - pdf = new double[1][numDurPdf][1][2 * numState]; // just one state and one stream - vsize = (2 * numState); - /* read pdfs (mean & variance) */ - // NOTE: Here (hts_engine v1.04) the order is different as before, here mean and variance are saved consecutively - for (i = 0; i < numDurPdf; i++) { - for (j = 0; j < numState; j++) { - pdf[0][i][0][j] = data_in.readFloat(); // read mean - pdf[0][i][0][j + numState] = data_in.readFloat(); // read variance - // System.out.println("durpdf[" + i + "]" + "[" + j + "]:" + pdf[0][i][0][j]); - } - } - data_in.close(); - data_in = null; - - } else if (fileFormat == PdfFileFormat.lf0) { // pdfFileName.contains("lf0.pdf") - /* ____________________________________________________________________ */ - /*-------------------- load pdfs for Log F0 --------------*/ - data_in = new DataInputStream(new BufferedInputStream(pdfStream)); - logger.debug("loadPdfs reading model of type " + fileFormat); - /* read the number of streams for f0 modeling */ - // lf0Stream = data_in.readInt(); - // vectorSize = lf0Stream; - numMSDFlag = data_in.readInt(); - numStream = data_in.readInt(); - vectorSize = data_in.readInt(); - - lf0Stream = numStream; - // System.out.println("loadPdfs: lf0stream = " + lf0stream); - - if (lf0Stream < 0) - throw new MaryConfigurationException("loadPdfs: #stream for log f0 part must be positive value."); - - /* read the number of pdfs for each state position */ - pdf = new double[numState][][][]; - numPdf = new int[numState]; - for (i = 0; i < numState; i++) { - numPdf[i] = data_in.readInt(); - logger.debug("loadPdfs: numPdf[state:" + i + "]=" + numPdf[i]); - if (numPdf[i] < 0) - throw new MaryConfigurationException("loadPdfs: #lf0 pdf at state " + i + " must be positive value."); - // System.out.println("nlf0pdf[" + i + "] = " + numPdf[i]); - /* Now i know the size of pdfs for lf0 [#states][#leaves][#streams][lf0_vectorsize] */ - /* lf0_vectorsize = 4: mean, variance, voiced weight, and unvoiced weight */ - /* so i can allocate memory for lf0pdf[][][] */ - pdf[i] = new double[numPdf[i]][lf0Stream][4]; - } - - /* read lf0 pdfs (mean, variance and weight). */ - for (i = 0; i < numState; i++) { - for (j = 0; j < numPdf[i]; j++) { - for (k = 0; k < lf0Stream; k++) { - for (l = 0; l < 4; l++) { - pdf[i][j][k][l] = data_in.readFloat(); - // System.out.format("pdf[%d][%d][%d][%d]=%f\n", i,j,k,l,pdf[i][j][k][l]); - } - // System.out.format("\n"); - // NOTE: Here (hts_engine v1.04) the order seem to be the same as before - /* pdf[i][j][k][0]; mean */ - /* pdf[i][j][k][1]; vari */ - vw = pdf[i][j][k][2]; /* voiced weight */ - uvw = pdf[i][j][k][3]; /* unvoiced weight */ - if (vw < 0.0 || uvw < 0.0 || vw + uvw < 0.99 || vw + uvw > 1.01) - throw new MaryConfigurationException("loadPdfs: voiced/unvoiced weights must be within 0.99 to 1.01."); - } - } - } - - data_in.close(); - data_in = null; - - } else if (fileFormat == PdfFileFormat.mgc || fileFormat == PdfFileFormat.str || fileFormat == PdfFileFormat.mag) { - // pdfFileName.contains("mgc.pdf") || - // pdfFileName.contains("str.pdf") || - // pdfFileName.contains("mag.pdf") - /* ___________________________________________________________________________ */ - /*-------------------- load pdfs for mgc, str or mag ------------------------*/ - data_in = new DataInputStream(new BufferedInputStream(pdfStream)); - logger.debug("loadPdfs reading model of type " + fileFormat); - /* read vector size for spectrum */ - - // numStream = 1; // just one stream for mgc, str, mag. This is just to have only one - // type of pdf vector for all posible pdf's - // vsize = data_in.readInt(); - // vectorSize = vsize; - numMSDFlag = data_in.readInt(); - numStream = data_in.readInt(); - vectorSize = data_in.readInt(); - - vsize = vectorSize; - // System.out.println("loadPdfs: vsize = " + vsize); - - if (vsize < 0) - throw new MaryConfigurationException("loadPdfs: vector size of pdf must be positive."); - - /* Now we need the number of pdf's for each state */ - pdf = new double[numState][][][]; - numPdf = new int[numState]; - for (i = 0; i < numState; i++) { - numPdf[i] = data_in.readInt(); - logger.debug("loadPdfs: numPdf[state:" + i + "]=" + numPdf[i]); - if (numPdf[i] < 0) - throw new MaryConfigurationException("loadPdfs: #pdf at state " + i + " must be positive value."); - // System.out.println("nmceppdf[" + i + "] = " + nmceppdf[i]); - /* Now i know the size of mceppdf[#states][#leaves][vectorsize] */ - /* so i can allocate memory for mceppdf[][][] */ - pdf[i] = new double[numPdf[i]][numStream][2 * vsize]; - } - - /* read pdfs (mean, variance). (2*vsize because mean and diag variance */ - /* are allocated in only one vector. */ - for (i = 0; i < numState; i++) { - for (j = 0; j < numPdf[i]; j++) { - /* - * for( k=0; k<(2*vsize); k++ ){ pdf[i][j][0][k] = data_in.readFloat(); // [0] corresponds to stream, in this - * case just one. //System.out.println("pdf["+ i + "][" + j + "][0][" + k + "] =" + pdf[i][j][0][k]); } - */ - // NOTE: Here (hts_engine v1.04) the order is different as before, here mean and variance are saved - // consecutively - // so now the pdf contains: mean[0], vari[0], mean[1], vari[1], etc... - for (k = 0; k < vsize; k++) { - pdf[i][j][0][k] = data_in.readFloat(); // [0] corresponds to stream, in this case just one. - // System.out.println("pdf["+ i + "][" + j + "][0][" + k + "] =" + pdf[i][j][0][k]); - pdf[i][j][0][k + vsize] = data_in.readFloat(); - } - } - } - data_in.close(); - data_in = null; - - } - - return pdf; - - } /* method loadPdfs */ - - public static void main(String[] args) throws IOException, InterruptedException { - /* configure log info */ - org.apache.log4j.BasicConfigurator.configure(); - - String contextFile = "/project/mary/marcela/openmary/lib/voices/hsmm-slt/cmu_us_arctic_slt_a0001.pfeats"; - Scanner context = new Scanner(new BufferedReader(new FileReader(contextFile))); - String strContext = ""; - while (context.hasNext()) { - strContext += context.nextLine(); - strContext += "\n"; - } - context.close(); - // System.out.println(strContext); - FeatureDefinition feaDef = new FeatureDefinition(new BufferedReader(new StringReader(strContext)), false); - - CART[] mgcTree = null; - int numStates = 5; - String trickyPhones = "/project/mary/marcela/openmary/lib/voices/hsmm-slt/trickyPhones.txt"; - String treefile = "/project/mary/marcela/openmary/lib/voices/hsmm-slt/tree-dur.inf"; - String pdffile = "/project/mary/marcela/openmary/lib/voices/hsmm-slt/dur.pdf"; - int vSize; - - // Check if there are tricky phones, and create a PhoneTranslator object - PhoneTranslator phTranslator = new PhoneTranslator(new FileInputStream(trickyPhones)); - - HTSCARTReader htsReader = new HTSCARTReader(); - try { - mgcTree = htsReader.load(numStates, new FileInputStream(treefile), new FileInputStream(pdffile), PdfFileFormat.dur, - feaDef, phTranslator); - vSize = htsReader.getVectorSize(); - System.out.println("loaded " + pdffile + " vector size=" + vSize); - } catch (Exception e) { - System.out.println(e.getMessage()); - } - - } - -} diff --git a/marytts-unitselection/src/main/java/marytts/cart/io/MaryCARTReader.java b/marytts-unitselection/src/main/java/marytts/cart/io/MaryCARTReader.java deleted file mode 100644 index 07ca4063d0..0000000000 --- a/marytts-unitselection/src/main/java/marytts/cart/io/MaryCARTReader.java +++ /dev/null @@ -1,402 +0,0 @@ -/** - * Copyright 2006 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.cart.io; - -import java.io.BufferedInputStream; -import java.io.ByteArrayInputStream; -import java.io.DataInput; -import java.io.DataInputStream; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.nio.ByteBuffer; -import java.nio.channels.FileChannel; -import java.util.Properties; - -import marytts.cart.CART; -import marytts.cart.DecisionNode; -import marytts.cart.LeafNode; -import marytts.cart.Node; -import marytts.exceptions.MaryConfigurationException; -import marytts.features.FeatureDefinition; -import marytts.util.data.MaryHeader; - -/** - * IO functions for CARTs in MaryCART format - * - * @author Marcela Charfuelan - */ -public class MaryCARTReader { - /** - * Load the cart from the given file - * - * @param fileName - * the file to load the cart from - * @param featDefinition - * the feature definition - * @param dummy - * unused, just here for compatibility with the FeatureFileIndexer. - * @throws IOException - * if a problem occurs while loading - */ - public CART load(String fileName) throws IOException, MaryConfigurationException { - FileInputStream fis = new FileInputStream(fileName); - try { - return loadFromStream(fis); - } finally { - fis.close(); - } - } - - /** - * Load the cart from the given file - * - * @param inStream - * the stream to load the cart from - * @param featDefinition - * the feature definition - * @param dummy - * unused, just here for compatibility with the FeatureFileIndexer. - * @throws IOException - * if a problem occurs while loading - */ - public CART loadFromStream(InputStream inStream) throws IOException, MaryConfigurationException { - // open the CART-File and read the header - DataInput raf = new DataInputStream(new BufferedInputStream(inStream)); - - MaryHeader maryHeader = new MaryHeader(raf); - if (!maryHeader.hasCurrentVersion()) { - throw new IOException("Wrong version of database file"); - } - if (maryHeader.getType() != MaryHeader.CARTS) { - throw new IOException("No CARTs file"); - } - - // Read properties - short propDataLength = raf.readShort(); - Properties props; - if (propDataLength == 0) { - props = null; - } else { - byte[] propsData = new byte[propDataLength]; - raf.readFully(propsData); - ByteArrayInputStream bais = new ByteArrayInputStream(propsData); - props = new Properties(); - props.load(bais); - bais.close(); - } - - // Read the feature definition - FeatureDefinition featureDefinition = new FeatureDefinition(raf); - - // read the decision nodes - int numDecNodes = raf.readInt(); // number of decision nodes - - // First we need to read all nodes into memory, then we can link them properly - // in terms of parent/child. - DecisionNode[] dns = new DecisionNode[numDecNodes]; - int[][] childIndexes = new int[numDecNodes][]; - for (int i = 0; i < numDecNodes; i++) { - // read one decision node - int featureNameIndex = raf.readInt(); - int nodeTypeNr = raf.readInt(); - DecisionNode.Type nodeType = DecisionNode.Type.values()[nodeTypeNr]; - int numChildren = 2; // for binary nodes - switch (nodeType) { - case BinaryByteDecisionNode: - int criterion = raf.readInt(); - dns[i] = new DecisionNode.BinaryByteDecisionNode(featureNameIndex, (byte) criterion, featureDefinition); - break; - case BinaryShortDecisionNode: - criterion = raf.readInt(); - dns[i] = new DecisionNode.BinaryShortDecisionNode(featureNameIndex, (short) criterion, featureDefinition); - break; - case BinaryFloatDecisionNode: - float floatCriterion = raf.readFloat(); - dns[i] = new DecisionNode.BinaryFloatDecisionNode(featureNameIndex, floatCriterion, featureDefinition); - break; - case ByteDecisionNode: - numChildren = raf.readInt(); - if (featureDefinition.getNumberOfValues(featureNameIndex) != numChildren) { - throw new IOException("Inconsistent cart file: feature " + featureDefinition.getFeatureName(featureNameIndex) - + " should have " + featureDefinition.getNumberOfValues(featureNameIndex) - + " values, but decision node " + i + " has only " + numChildren + " child nodes"); - } - dns[i] = new DecisionNode.ByteDecisionNode(featureNameIndex, numChildren, featureDefinition); - break; - case ShortDecisionNode: - numChildren = raf.readInt(); - if (featureDefinition.getNumberOfValues(featureNameIndex) != numChildren) { - throw new IOException("Inconsistent cart file: feature " + featureDefinition.getFeatureName(featureNameIndex) - + " should have " + featureDefinition.getNumberOfValues(featureNameIndex) - + " values, but decision node " + i + " has only " + numChildren + " child nodes"); - } - dns[i] = new DecisionNode.ShortDecisionNode(featureNameIndex, numChildren, featureDefinition); - } - // now read the children, indexes only: - childIndexes[i] = new int[numChildren]; - for (int k = 0; k < numChildren; k++) { - childIndexes[i][k] = raf.readInt(); - } - } - - // read the leaves - int numLeafNodes = raf.readInt(); // number of leaves, it does not include empty leaves - LeafNode[] lns = new LeafNode[numLeafNodes]; - - for (int j = 0; j < numLeafNodes; j++) { - // read one leaf node - int leafTypeNr = raf.readInt(); - LeafNode.LeafType leafNodeType = LeafNode.LeafType.values()[leafTypeNr]; - switch (leafNodeType) { - case IntArrayLeafNode: - int numData = raf.readInt(); - int[] data = new int[numData]; - for (int d = 0; d < numData; d++) { - data[d] = raf.readInt(); - } - lns[j] = new LeafNode.IntArrayLeafNode(data); - break; - case FloatLeafNode: - float stddev = raf.readFloat(); - float mean = raf.readFloat(); - lns[j] = new LeafNode.FloatLeafNode(new float[] { stddev, mean }); - break; - case IntAndFloatArrayLeafNode: - case StringAndFloatLeafNode: - int numPairs = raf.readInt(); - int[] ints = new int[numPairs]; - float[] floats = new float[numPairs]; - for (int d = 0; d < numPairs; d++) { - ints[d] = raf.readInt(); - floats[d] = raf.readFloat(); - } - if (leafNodeType == LeafNode.LeafType.IntAndFloatArrayLeafNode) - lns[j] = new LeafNode.IntAndFloatArrayLeafNode(ints, floats); - else - lns[j] = new LeafNode.StringAndFloatLeafNode(ints, floats); - break; - case FeatureVectorLeafNode: - throw new IllegalArgumentException("Reading feature vector leaf nodes is not yet implemented"); - case PdfLeafNode: - throw new IllegalArgumentException("Reading pdf leaf nodes is not yet implemented"); - } - } - - // Now, link up the decision nodes with their daughters - for (int i = 0; i < numDecNodes; i++) { - for (int k = 0; k < childIndexes[i].length; k++) { - int childIndex = childIndexes[i][k]; - if (childIndex < 0) { // a decision node - assert -childIndex - 1 < numDecNodes; - dns[i].addDaughter(dns[-childIndex - 1]); - } else if (childIndex > 0) { // a leaf node - dns[i].addDaughter(lns[childIndex - 1]); - } else { // == 0, an empty leaf - dns[i].addDaughter(null); - } - } - } - - Node rootNode; - if (dns.length > 0) { - rootNode = dns[0]; - // Now count all data once, so that getNumberOfData() - // will return the correct figure. - ((DecisionNode) rootNode).countData(); - } else if (lns.length > 0) { - rootNode = lns[0]; // single-leaf tree... - } else { - rootNode = null; - } - - // set the rootNode as the rootNode of cart - return new CART(rootNode, featureDefinition, props); - } - - /** - * Load the cart from the given file - * - * @param fileName - * the file to load the cart from - * @param featDefinition - * the feature definition - * @param dummy - * unused, just here for compatibility with the FeatureFileIndexer. - * @throws IOException - * if a problem occurs while loading - */ - private CART loadFromByteBuffer(String fileName) throws IOException, MaryConfigurationException { - // open the CART-File and read the header - FileInputStream fis = new FileInputStream(fileName); - FileChannel fc = fis.getChannel(); - ByteBuffer bb = fc.map(FileChannel.MapMode.READ_ONLY, 0, fc.size()); - fis.close(); - - MaryHeader maryHeader = new MaryHeader(bb); - if (!maryHeader.hasCurrentVersion()) { - throw new IOException("Wrong version of database file"); - } - if (maryHeader.getType() != MaryHeader.CARTS) { - throw new IOException("No CARTs file"); - } - - // Read properties - short propDataLength = bb.getShort(); - Properties props; - if (propDataLength == 0) { - props = null; - } else { - byte[] propsData = new byte[propDataLength]; - bb.get(propsData); - ByteArrayInputStream bais = new ByteArrayInputStream(propsData); - props = new Properties(); - props.load(bais); - bais.close(); - } - - // Read the feature definition - FeatureDefinition featureDefinition = new FeatureDefinition(bb); - - // read the decision nodes - int numDecNodes = bb.getInt(); // number of decision nodes - - // First we need to read all nodes into memory, then we can link them properly - // in terms of parent/child. - DecisionNode[] dns = new DecisionNode[numDecNodes]; - int[][] childIndexes = new int[numDecNodes][]; - for (int i = 0; i < numDecNodes; i++) { - // read one decision node - int featureNameIndex = bb.getInt(); - int nodeTypeNr = bb.getInt(); - DecisionNode.Type nodeType = DecisionNode.Type.values()[nodeTypeNr]; - int numChildren = 2; // for binary nodes - switch (nodeType) { - case BinaryByteDecisionNode: - int criterion = bb.getInt(); - dns[i] = new DecisionNode.BinaryByteDecisionNode(featureNameIndex, (byte) criterion, featureDefinition); - break; - case BinaryShortDecisionNode: - criterion = bb.getInt(); - dns[i] = new DecisionNode.BinaryShortDecisionNode(featureNameIndex, (short) criterion, featureDefinition); - break; - case BinaryFloatDecisionNode: - float floatCriterion = bb.getFloat(); - dns[i] = new DecisionNode.BinaryFloatDecisionNode(featureNameIndex, floatCriterion, featureDefinition); - break; - case ByteDecisionNode: - numChildren = bb.getInt(); - if (featureDefinition.getNumberOfValues(featureNameIndex) != numChildren) { - throw new IOException("Inconsistent cart file: feature " + featureDefinition.getFeatureName(featureNameIndex) - + " should have " + featureDefinition.getNumberOfValues(featureNameIndex) - + " values, but decision node " + i + " has only " + numChildren + " child nodes"); - } - dns[i] = new DecisionNode.ByteDecisionNode(featureNameIndex, numChildren, featureDefinition); - break; - case ShortDecisionNode: - numChildren = bb.getInt(); - if (featureDefinition.getNumberOfValues(featureNameIndex) != numChildren) { - throw new IOException("Inconsistent cart file: feature " + featureDefinition.getFeatureName(featureNameIndex) - + " should have " + featureDefinition.getNumberOfValues(featureNameIndex) - + " values, but decision node " + i + " has only " + numChildren + " child nodes"); - } - dns[i] = new DecisionNode.ShortDecisionNode(featureNameIndex, numChildren, featureDefinition); - } - // now read the children, indexes only: - childIndexes[i] = new int[numChildren]; - for (int k = 0; k < numChildren; k++) { - childIndexes[i][k] = bb.getInt(); - } - } - - // read the leaves - int numLeafNodes = bb.getInt(); // number of leaves, it does not include empty leaves - LeafNode[] lns = new LeafNode[numLeafNodes]; - - for (int j = 0; j < numLeafNodes; j++) { - // read one leaf node - int leafTypeNr = bb.getInt(); - LeafNode.LeafType leafNodeType = LeafNode.LeafType.values()[leafTypeNr]; - switch (leafNodeType) { - case IntArrayLeafNode: - int numData = bb.getInt(); - int[] data = new int[numData]; - for (int d = 0; d < numData; d++) { - data[d] = bb.getInt(); - } - lns[j] = new LeafNode.IntArrayLeafNode(data); - break; - case FloatLeafNode: - float stddev = bb.getFloat(); - float mean = bb.getFloat(); - lns[j] = new LeafNode.FloatLeafNode(new float[] { stddev, mean }); - break; - case IntAndFloatArrayLeafNode: - case StringAndFloatLeafNode: - int numPairs = bb.getInt(); - int[] ints = new int[numPairs]; - float[] floats = new float[numPairs]; - for (int d = 0; d < numPairs; d++) { - ints[d] = bb.getInt(); - floats[d] = bb.getFloat(); - } - if (leafNodeType == LeafNode.LeafType.IntAndFloatArrayLeafNode) - lns[j] = new LeafNode.IntAndFloatArrayLeafNode(ints, floats); - else - lns[j] = new LeafNode.StringAndFloatLeafNode(ints, floats); - break; - case FeatureVectorLeafNode: - throw new IllegalArgumentException("Reading feature vector leaf nodes is not yet implemented"); - case PdfLeafNode: - throw new IllegalArgumentException("Reading pdf leaf nodes is not yet implemented"); - } - } - - // Now, link up the decision nodes with their daughters - for (int i = 0; i < numDecNodes; i++) { - for (int k = 0; k < childIndexes[i].length; k++) { - int childIndex = childIndexes[i][k]; - if (childIndex < 0) { // a decision node - assert -childIndex - 1 < numDecNodes; - dns[i].addDaughter(dns[-childIndex - 1]); - } else if (childIndex > 0) { // a leaf node - dns[i].addDaughter(lns[childIndex - 1]); - } else { // == 0, an empty leaf - dns[i].addDaughter(null); - } - } - } - - Node rootNode; - if (dns.length > 0) { - rootNode = dns[0]; - // Now count all data once, so that getNumberOfData() - // will return the correct figure. - ((DecisionNode) rootNode).countData(); - } else if (lns.length > 0) { - rootNode = lns[0]; // single-leaf tree... - } else { - rootNode = null; - } - - // set the rootNode as the rootNode of cart - return new CART(rootNode, featureDefinition, props); - } -} diff --git a/marytts-unitselection/src/main/java/marytts/cart/io/MaryCARTWriter.java b/marytts-unitselection/src/main/java/marytts/cart/io/MaryCARTWriter.java deleted file mode 100644 index 2216737708..0000000000 --- a/marytts-unitselection/src/main/java/marytts/cart/io/MaryCARTWriter.java +++ /dev/null @@ -1,345 +0,0 @@ -/** - * Copyright 2006 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.cart.io; - -import java.io.BufferedOutputStream; -import java.io.ByteArrayOutputStream; -import java.io.DataOutput; -import java.io.DataOutputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.PrintWriter; -import java.util.Properties; - -import marytts.cart.CART; -import marytts.cart.DecisionNode; -import marytts.cart.LeafNode; -import marytts.cart.Node; -import marytts.cart.DecisionNode.BinaryByteDecisionNode; -import marytts.cart.DecisionNode.BinaryFloatDecisionNode; -import marytts.cart.DecisionNode.BinaryShortDecisionNode; -import marytts.cart.LeafNode.FeatureVectorLeafNode; -import marytts.cart.LeafNode.FloatLeafNode; -import marytts.cart.LeafNode.IntAndFloatArrayLeafNode; -import marytts.cart.LeafNode.IntArrayLeafNode; -import marytts.cart.LeafNode.LeafType; -import marytts.features.FeatureVector; -import marytts.util.MaryUtils; -import marytts.util.data.MaryHeader; - -import org.apache.log4j.Logger; - -/** - * IO functions for CARTs in MaryCART format - * - * @author Marcela Charfuelan - */ -public class MaryCARTWriter { - - protected Logger logger = MaryUtils.getLogger(this.getClass().getName()); - - /** - * Dump the CARTs in MaryCART format - * - * @param destDir - * the destination directory - */ - public void dumpMaryCART(CART cart, String destFile) throws IOException { - if (cart == null) - throw new NullPointerException("Cannot dump null CART"); - if (destFile == null) - throw new NullPointerException("No destination file"); - - logger.debug("Dumping CART in MaryCART format to " + destFile + " ..."); - - // Open the destination file (cart.bin) and output the header - DataOutputStream out = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(destFile))); - // create new CART-header and write it to output file - MaryHeader hdr = new MaryHeader(MaryHeader.CARTS); - hdr.writeTo(out); - - Properties props = cart.getProperties(); - if (props == null) { - out.writeShort(0); - } else { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - props.store(baos, null); - byte[] propData = baos.toByteArray(); - out.writeShort(propData.length); - out.write(propData); - } - - // feature definition - cart.getFeatureDefinition().writeBinaryTo(out); - - // dump CART - dumpBinary(cart.getRootNode(), out); - - // finish - out.close(); - logger.debug(" ... done\n"); - } - - public void toTextOut(CART cart, PrintWriter pw) throws IOException { - try { - int id[] = new int[2]; - id[0] = 0; // number of decision nodes - id[1] = 0; // number of leaf nodes - - // System.out.println("Total number of nodes:" + rootNode.getNumberOfNodes()); - setUniqueNodeId(cart.getRootNode(), id); - pw.println("Num decision nodes= " + id[0] + " Num leaf nodes= " + id[1]); - printDecisionNodes(cart.getRootNode(), null, pw); - pw.println("\n----------------\n"); - printLeafNodes(cart.getRootNode(), null, pw); - - pw.flush(); - pw.close(); - } catch (IOException ioe) { - IOException newIOE = new IOException("Error dumping CART to standard output"); - newIOE.initCause(ioe); - throw newIOE; - } - } - - private void setUniqueNodeId(Node node, int id[]) throws IOException { - - int thisIdNode; - String leafstr = ""; - - // if the node is decision node - if (node.getNumberOfNodes() > 1) { - assert node instanceof DecisionNode; - DecisionNode decNode = (DecisionNode) node; - id[0]--; - decNode.setUniqueDecisionNodeId(id[0]); - String strNode = ""; - // this.decNodeStr = ""; - thisIdNode = id[0]; - - // add Ids to the daughters - for (int i = 0; i < decNode.getNumberOfDaugthers(); i++) { - setUniqueNodeId(decNode.getDaughter(i), id); - } - - } else { // the node is a leaf node - assert node instanceof LeafNode; - LeafNode leaf = (LeafNode) node; - if (leaf.isEmpty()) { - leaf.setUniqueLeafId(0); - } else { - id[1]++; - leaf.setUniqueLeafId(id[1]); - } - - } - - } - - private void dumpBinary(Node rootNode, DataOutput os) throws IOException { - try { - - int id[] = new int[2]; - id[0] = 0; // number of decision nodes - id[1] = 0; // number of leaf nodes - // first add unique identifiers to decision nodes and leaf nodes - setUniqueNodeId(rootNode, id); - - // write the number of decision nodes - os.writeInt(Math.abs(id[0])); - // lines that start with a negative number are decision nodes - printDecisionNodes(rootNode, os, null); - - // write the number of leaves. - os.writeInt(id[1]); - // lines that start with id are leaf nodes - printLeafNodes(rootNode, (DataOutputStream) os, null); - - } catch (IOException ioe) { - IOException newIOE = new IOException("Error dumping CART to output stream"); - newIOE.initCause(ioe); - throw newIOE; - } - } - - private void printDecisionNodes(Node node, DataOutput out, PrintWriter pw) throws IOException { - if (!(node instanceof DecisionNode)) - return; // nothing to do here - - DecisionNode decNode = (DecisionNode) node; - int id = decNode.getUniqueDecisionNodeId(); - String nodeDefinition = decNode.getNodeDefinition(); - int featureIndex = decNode.getFeatureIndex(); - DecisionNode.Type nodeType = decNode.getDecisionNodeType(); - - if (out != null) { - // dump in binary form to output - out.writeInt(featureIndex); - out.writeInt(nodeType.ordinal()); - // Now, questionValue, which depends on nodeType - switch (nodeType) { - case BinaryByteDecisionNode: - out.writeInt(((BinaryByteDecisionNode) decNode).getCriterionValueAsByte()); - assert decNode.getNumberOfDaugthers() == 2; - break; - case BinaryShortDecisionNode: - out.writeInt(((BinaryShortDecisionNode) decNode).getCriterionValueAsShort()); - assert decNode.getNumberOfDaugthers() == 2; - break; - case BinaryFloatDecisionNode: - out.writeFloat(((BinaryFloatDecisionNode) decNode).getCriterionValueAsFloat()); - assert decNode.getNumberOfDaugthers() == 2; - break; - case ByteDecisionNode: - case ShortDecisionNode: - out.writeInt(decNode.getNumberOfDaugthers()); - } - - // The child nodes - for (int i = 0, n = decNode.getNumberOfDaugthers(); i < n; i++) { - Node daughter = decNode.getDaughter(i); - if (daughter instanceof DecisionNode) { - out.writeInt(((DecisionNode) daughter).getUniqueDecisionNodeId()); - } else { - assert daughter instanceof LeafNode; - out.writeInt(((LeafNode) daughter).getUniqueLeafId()); - } - } - } - if (pw != null) { - // dump to print writer - StringBuilder strNode = new StringBuilder(id + " " + nodeDefinition); - for (int i = 0, n = decNode.getNumberOfDaugthers(); i < n; i++) { - strNode.append(" "); - Node daughter = decNode.getDaughter(i); - if (daughter instanceof DecisionNode) { - strNode.append(((DecisionNode) daughter).getUniqueDecisionNodeId()); - } else { - assert daughter instanceof LeafNode; - strNode.append("id").append(((LeafNode) daughter).getUniqueLeafId()); - } - } - pw.println(strNode.toString()); - } - // add the daughters - for (int i = 0; i < ((DecisionNode) node).getNumberOfDaugthers(); i++) { - if (((DecisionNode) node).getDaughter(i).getNumberOfNodes() > 1) - printDecisionNodes(((DecisionNode) node).getDaughter(i), out, pw); - } - } - - /** This function will print the leaf nodes only, but it goes through all the decision nodes. */ - private void printLeafNodes(Node node, DataOutput out, PrintWriter pw) throws IOException { - // If the node does not have leaves then it just return. - // I we are in a decision node then print the leaves of the daughters. - Node nextNode; - if (node.getNumberOfNodes() > 1) { - assert node instanceof DecisionNode; - DecisionNode decNode = (DecisionNode) node; - for (int i = 0; i < decNode.getNumberOfDaugthers(); i++) { - nextNode = decNode.getDaughter(i); - printLeafNodes(nextNode, out, pw); - } - } else { - assert node instanceof LeafNode; - LeafNode leaf = (LeafNode) node; - if (leaf.getUniqueLeafId() == 0) // empty leaf, do not write - return; - LeafType leafType = leaf.getLeafNodeType(); - if (leafType == LeafType.FeatureVectorLeafNode) { - leafType = LeafType.IntArrayLeafNode; - // save feature vector leaf nodes as int array leaf nodes - } - if (out != null) { - // Leaf node type - out.writeInt(leafType.ordinal()); - } - if (pw != null) { - pw.print("id" + leaf.getUniqueLeafId() + " " + leafType); - } - switch (leaf.getLeafNodeType()) { - case IntArrayLeafNode: - int data[] = ((IntArrayLeafNode) leaf).getIntData(); - // Number of data points following: - if (out != null) - out.writeInt(data.length); - if (pw != null) - pw.print(" " + data.length); - // for each index, write the index - for (int i = 0; i < data.length; i++) { - if (out != null) - out.writeInt(data[i]); - if (pw != null) - pw.print(" " + data[i]); - } - break; - case FloatLeafNode: - float stddev = ((FloatLeafNode) leaf).getStDeviation(); - float mean = ((FloatLeafNode) leaf).getMean(); - if (out != null) { - out.writeFloat(stddev); - out.writeFloat(mean); - } - if (pw != null) { - pw.print(" 1 " + stddev + " " + mean); - } - break; - case IntAndFloatArrayLeafNode: - case StringAndFloatLeafNode: - int data1[] = ((IntAndFloatArrayLeafNode) leaf).getIntData(); - float floats[] = ((IntAndFloatArrayLeafNode) leaf).getFloatData(); - // Number of data points following: - if (out != null) - out.writeInt(data1.length); - if (pw != null) - pw.print(" " + data1.length); - // for each index, write the index and then its float - for (int i = 0; i < data1.length; i++) { - if (out != null) { - out.writeInt(data1[i]); - out.writeFloat(floats[i]); - } - if (pw != null) - pw.print(" " + data1[i] + " " + floats[i]); - } - break; - case FeatureVectorLeafNode: - FeatureVector fv[] = ((FeatureVectorLeafNode) leaf).getFeatureVectors(); - // Number of data points following: - if (out != null) - out.writeInt(fv.length); - if (pw != null) - pw.print(" " + fv.length); - // for each feature vector, write the index - for (int i = 0; i < fv.length; i++) { - if (out != null) - out.writeInt(fv[i].getUnitIndex()); - if (pw != null) - pw.print(" " + fv[i].getUnitIndex()); - } - break; - case PdfLeafNode: - throw new IllegalArgumentException("Writing of pdf leaf nodes not yet implemented"); - } - if (pw != null) - pw.println(); - } - } -} diff --git a/marytts-unitselection/src/main/java/marytts/cart/io/WagonCARTReader.java b/marytts-unitselection/src/main/java/marytts/cart/io/WagonCARTReader.java deleted file mode 100644 index 17076fbcd7..0000000000 --- a/marytts-unitselection/src/main/java/marytts/cart/io/WagonCARTReader.java +++ /dev/null @@ -1,607 +0,0 @@ -/** - * Copyright 2006 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.cart.io; - -import java.io.BufferedInputStream; -import java.io.BufferedReader; -import java.io.DataInput; -import java.io.DataInputStream; -import java.io.EOFException; -import java.io.FileInputStream; -import java.io.IOException; -import java.util.StringTokenizer; - -import marytts.cart.DecisionNode; -import marytts.cart.LeafNode; -import marytts.cart.Node; -import marytts.exceptions.MaryConfigurationException; -import marytts.features.FeatureDefinition; -import marytts.features.FeatureVector; -import marytts.util.data.MaryHeader; - -/** - * IO functions for CARTs in WagonCART format - * - * @author Anna Hunecke, Marc Schröder, Marcela Charfuelan - */ -public class WagonCARTReader { - - private Node rootNode; - private Node lastNode; - - // knows the index numbers and types of the features used in DecisionNodes - private FeatureDefinition featDef; - - private int openBrackets; - - // Since it is not known from the wagon file lines which kind of leaves - // should be read, a leafType argument should be provided when creating - // this class. - private LeafNode.LeafType leafType; - - // added because StringCART - private int targetFeature; - - /** - * When creating a WagonCARTReader provide a tree type - * - * @param treeType - * ClasificationTree, ExtendedClassificationTree, RegressionTree, or TopLevelTree. - * - *

- * ClasificationTree --> IntArrayLeafNode - *

- * ExtendedClassificationTree --> IntAndFloatArrayLeafNode - *

- * RegressionTree --> FloatLeafNode - *

- * TopLevelTree --> FeatureVectorLeafNode - *

- * StringCART --> StringAndFloatLeafNode - */ - public WagonCARTReader(LeafNode.LeafType leafType) { - this.leafType = leafType; - } - - /** - * For a line representing a leaf in Wagon format, create a leaf. This method decides which implementation of LeafNode is - * used, i.e. which data format is appropriate. Lines are of the form (( )...( )) 0)) - * - * @param line - * a line from a wagon cart file, representing a leaf - * @return a leaf node representing the line. - */ - protected LeafNode createLeafNode(String line) { - if (leafType == LeafNode.LeafType.IntArrayLeafNode) - return (createIntArrayLeafNode(line)); - else if (leafType == LeafNode.LeafType.IntAndFloatArrayLeafNode) - return (createIntAndFloatArrayLeafNode(line)); - else if (leafType == LeafNode.LeafType.FloatLeafNode) - return (createFloatLeafNode(line)); - else if (leafType == LeafNode.LeafType.FeatureVectorLeafNode) - return (createFeatureVectorLeafNode(line)); - else if (leafType == LeafNode.LeafType.StringAndFloatLeafNode) - return (createStringAndFloatLeafNode(line)); - else - return null; - } - - // in case of using the reader more than once for different root nodes. - private void cleadReader() { - rootNode = null; - lastNode = null; - featDef = null; - openBrackets = 0; - } - - /** - * - * This loads a cart from a wagon tree in textual format, from a reader. - * - * @param reader - * the Reader providing the wagon tree - * @param featDefinition - * @throws IOException - */ - public Node load(BufferedReader reader, FeatureDefinition featDefinition) throws IOException { - cleadReader(); - featDef = featDefinition; - openBrackets = 0; - String line = reader.readLine(); - if (line.equals("")) {// first line is empty, read again - line = reader.readLine(); - } - // each line corresponds to a node - // for each line - while (line != null) { - if (!line.startsWith(";;") && !line.equals("")) { - // parse the line and add the node - - parseAndAdd(line); - } - line = reader.readLine(); - } - // make sure we closed as many brackets as we opened - if (openBrackets != 0) { - throw new IOException("Error loading CART: bracket mismatch"); - } - // Now count all data once, so that getNumberOfData() - // will return the correct figure. - if (rootNode instanceof DecisionNode) - ((DecisionNode) rootNode).countData(); - - return rootNode; - } - - /** - * Load the cart from the given file - * - * @param fileName - * the file to load the cart from - * @param featDefinition - * the feature definition - * @param dummy - * unused, just here for compatibility with the FeatureFileIndexer. - * @throws IOException - * if a problem occurs while loading - */ - // TODO: CHECK! do we need that String[] dummy??? - public Node load(String fileName, FeatureDefinition featDefinition, String[] dummy) throws IOException, - MaryConfigurationException { - cleadReader(); - // System.out.println("Loading file"); - // open the CART-File and read the header - DataInput raf = new DataInputStream(new BufferedInputStream(new FileInputStream(fileName))); - MaryHeader maryHeader = new MaryHeader(raf); - if (!maryHeader.hasCurrentVersion()) { - throw new IOException("Wrong version of database file"); - } - if (maryHeader.getType() != MaryHeader.CARTS) { - throw new IOException("No CARTs file"); - } - // System.out.println("Reading CART"); - // discard number of CARTs and CART name - // TODO: Change format of CART-File - int numNodes = raf.readInt(); - raf.readUTF(); - - // load the CART - featDef = featDefinition; - // get the backtrace information - openBrackets = 0; - // Not elegant, but robust - try { - while (true) { - // parse the line and add the node - int length = raf.readInt(); - char[] cartChars = new char[length]; - for (int i = 0; i < length; i++) { - cartChars[i] = raf.readChar(); - } - String cart = new String(cartChars); - // System.out.println(cart); - parseAndAdd(cart); - } - } catch (EOFException eof) { - } - - // make sure we closed as many brackets as we opened - if (openBrackets != 0) { - throw new IOException("Error loading CART: bracket mismatch: " + openBrackets); - } - // Now count all data once, so that getNumberOfData() - // will return the correct figure. - if (rootNode instanceof DecisionNode) - ((DecisionNode) rootNode).countData(); - // System.out.println("Done"); - - return rootNode; - } - - /** - * Creates a node from the given input line and add it to the CART. - * - * @param line - * a line of input to parse - * @throws IOException - * if the line has an unexpected format - */ - private void parseAndAdd(String line) throws IOException { - // remove whitespace - line = line.trim(); - // at beginning of String there should be at least two opening brackets - if (!(line.startsWith("(("))) { - throw new IOException("Invalid input line for CART: " + line); - } - if (Character.isLetter(line.charAt(2)) && !line.substring(2, 6).equals("nan ")) { // we have a node - openBrackets++; // do not count first bracket - - // get the properties of the node - StringTokenizer tokenizer = new StringTokenizer(line, " "); - String feature = tokenizer.nextToken().substring(2); - String type = tokenizer.nextToken(); - String value = tokenizer.nextToken(); - value = value.substring(0, value.length() - 1); - // some values are enclosed in double quotes: - if (value.startsWith("\"") && value.endsWith("\"") && value.length() > 2) - value = value.substring(1, value.length() - 1); - - // a literal double quote is escaped by backslash, so unescape it: - if (value.contains("\\\"")) { - value = value.replaceAll("\\\\\"", "\""); - } - - // build new node depending on type - - Node nextNode; - try { - if (type.equals("is")) { - if (featDef.isByteFeature(feature)) { - nextNode = new DecisionNode.BinaryByteDecisionNode(feature, value, featDef); - } else { - nextNode = new DecisionNode.BinaryShortDecisionNode(feature, value, featDef); - } - } else { - if (type.equals("<")) { - nextNode = new DecisionNode.BinaryFloatDecisionNode(feature, Float.parseFloat(value), featDef); - } else { - if (type.equals("isShortOf")) { - nextNode = new DecisionNode.ShortDecisionNode(feature, Integer.parseInt(value), featDef); - } else { - if (type.equals("isByteOf")) { - nextNode = new DecisionNode.ByteDecisionNode(feature, Integer.parseInt(value), featDef); - } else { - throw new IOException("Unknown node type : " + type); - } - } - } - } - - } catch (Exception exc) { - throw new RuntimeException("Cannot create decision node for cart line: '" + line + "'", exc); - } - - if (lastNode != null) { - // this node is the daughter of the last node - ((DecisionNode) lastNode).addDaughter(nextNode); - } else { - // this is the rootNode - rootNode = nextNode; - nextNode.setIsRoot(true); - } - - // go one step down - lastNode = nextNode; - - } else { // we have a leaf - - Node nextNode = createLeafNode(line); - - // set the relations of this node to the others - if (lastNode == null) { // this node is the root - rootNode = nextNode; - nextNode.setIsRoot(true); - } else { // this node is a daughter of lastNode - ((DecisionNode) lastNode).addDaughter(nextNode); - } - - // look at the bracketing at the end of the line: - // get the last token out of the tokenizer - StringTokenizer tokenizer = new StringTokenizer(line, " "); - for (int i = 0, numTokens = tokenizer.countTokens(); i < numTokens - 1; i++) { - tokenizer.nextToken(); - } - String lastToken = tokenizer.nextToken(); - - // lastToken should look like "0))" - // more than two brackets mean that this is - // the last daughter of one or more nodes - int length = lastToken.length(); - // start looking at the characters after "0))" - int index = lastToken.indexOf(')') + 2; - - while (index < length) { // while we have more characters - char nextChar = lastToken.charAt(index); - if (nextChar == ')') { - // if the next character is a closing bracket - openBrackets--; - // this is the last daughter of lastNode, - // try going one step up - if (lastNode.isRoot()) { - if (index + 1 != length) { - // lastNode should not be the root, - // unless we are at the last bracket - throw new IOException("Too many closing brackets in line " + line); - } - } else { // you can go one step up - nextNode = lastNode; - lastNode = lastNode.getMother(); - } - } else { - // nextChar is not a closing bracket; - // something went wrong here - throw new IOException("Expected closing bracket in line " + line + ", but found " + nextChar); - } - index++; - } - // for debugging - if (nextNode != null) { - int nodeIndex = nextNode.getNodeIndex(); - } - - } - } - - protected LeafNode createIntArrayLeafNode(String line) { - StringTokenizer tok = new StringTokenizer(line, " "); - // read the indices from the tokenized String - int numTokens = tok.countTokens(); - int index = 0; - // The data to be saved in the leaf node: - int[] indices; - if (numTokens == 2) { // we do not have any indices - // discard useless token - tok.nextToken(); - indices = new int[0]; - } else { - indices = new int[(numTokens - 1) / 2]; - - while (index * 2 < numTokens - 1) { // while we are not at the - // last token - String nextToken = tok.nextToken(); - if (index == 0) { - // we are at first token, discard all open brackets - nextToken = nextToken.substring(4); - } else { - // we are not at first token, only one open bracket - nextToken = nextToken.substring(1); - } - // store the index of the unit - indices[index] = Integer.parseInt(nextToken); - // discard next token - tok.nextToken(); - // increase index - index++; - } - } - return new LeafNode.IntArrayLeafNode(indices); - } - - protected LeafNode createIntAndFloatArrayLeafNode(String line) { - StringTokenizer tok = new StringTokenizer(line, " "); - // read the indices from the tokenized String - int numTokens = tok.countTokens(); - int index = 0; - // The data to be saved in the leaf node: - int[] indices; - // The floats to be saved in the leaf node: - float[] probs; - - // System.out.println("Line: "+line+", numTokens: "+numTokens); - - if (numTokens == 2) { // we do not have any indices - // discard useless token - tok.nextToken(); - indices = new int[0]; - probs = new float[0]; - } else { - indices = new int[(numTokens - 1) / 2]; - // same length - probs = new float[indices.length]; - - while (index * 2 < numTokens - 1) { - String token = tok.nextToken(); - if (index == 0) { - token = token.substring(4); - } else { - token = token.substring(1); - } - // System.out.println("int-token: "+token); - indices[index] = Integer.parseInt(token); - - token = tok.nextToken(); - int lastIndex = token.length() - 1; - if ((index * 2) == (numTokens - 3)) { - token = token.substring(0, lastIndex - 1); - if (token.equals("inf")) { - probs[index] = 10000; - index++; - continue; - } - if (token.equals("nan")) { - probs[index] = -1; - index++; - continue; - } - } else { - token = token.substring(0, lastIndex); - if (token.equals("inf")) { - probs[index] = 1000000; - index++; - continue; - } - if (token.equals("nan")) { - probs[index] = -1; - index++; - continue; - } - } - // System.out.println("float-token: "+token); - probs[index] = Float.parseFloat(token); - index++; - } // end while - - } // end if - - return new LeafNode.IntAndFloatArrayLeafNode(indices, probs); - } - - protected LeafNode createFloatLeafNode(String line) { - StringTokenizer tok = new StringTokenizer(line, " "); - // read the indices from the tokenized String - int numTokens = tok.countTokens(); - if (numTokens != 2) { // we need exactly one value pair - throw new IllegalArgumentException("Expected two tokens in line, got " + numTokens + ": '" + line + "'"); - } - - // The data to be saved in the leaf node: - float[] data = new float[2]; // stddev and mean; - String nextToken = tok.nextToken(); - nextToken = nextToken.substring(2); - try { - data[0] = Float.parseFloat(nextToken); - } catch (NumberFormatException nfe) { - data[0] = 0; // cannot make sense of the standard deviation - } - nextToken = tok.nextToken(); - nextToken = nextToken.substring(0, nextToken.indexOf(")")); - try { - data[1] = Float.parseFloat(nextToken); - } catch (NumberFormatException nfe) { - data[1] = 0; - } - return new LeafNode.FloatLeafNode(data); - } - - protected LeafNode createFeatureVectorLeafNode(String line) { - - StringTokenizer tok = new StringTokenizer(line, " "); - // read the indices from the tokenized String - int numTokens = tok.countTokens(); - int index = 0; - // The data to be saved in the leaf node: - if (numTokens != 2) { - // leaf is not empty -> error - throw new Error("Leaf in line " + line + " is not empty"); - } - // discard useless token - tok.nextToken(); - return new LeafNode.FeatureVectorLeafNode(); - - } - - /** - * Fill the FeatureVector leafs of a tree with the given feature vectors. This function is only used in TopLeavelTree. - * - * @param root - * node of the tree. - * @param featureVectors - * the feature vectors. - */ - public void fillLeafs(Node root, FeatureVector[] featureVectors) { - if (leafType == LeafNode.LeafType.FeatureVectorLeafNode) { - rootNode = root; - Node currentNode = rootNode; - Node prevNode = null; - - // loop trough the feature vectors - for (int i = 0; i < featureVectors.length; i++) { - currentNode = rootNode; - prevNode = null; - FeatureVector featureVector = featureVectors[i]; - // logger.debug("Starting cart at "+nodeIndex); - while (!(currentNode instanceof LeafNode)) { - // while we have not reached the bottom, - // get the next node based on the features of the target - prevNode = currentNode; - currentNode = ((DecisionNode) currentNode).getNextNode(featureVector); - // logger.debug(decision.toString() + " result '"+ - // decision.findFeature(item) + "' => "+ nodeIndex); - } - // add the feature vector to the leaf node - ((LeafNode.FeatureVectorLeafNode) currentNode).addFeatureVector(featureVector); - } - } else - throw new IllegalArgumentException("The leaves of this tree are not FeatureVectorLeafNode."); - - } - - protected LeafNode createStringAndFloatLeafNode(String line) { - // Note: this code is identical to createIntAndFloatArrayLeafNode(), - // except for the last line. - StringTokenizer tok = new StringTokenizer(line, " "); - // read the indices from the tokenized String - int numTokens = tok.countTokens(); - int index = 0; - // The data to be saved in the leaf node: - int[] indices; - // The floats to be saved in the leaf node: - float[] probs; - - // System.out.println("Line: "+line+", numTokens: "+numTokens); - - if (numTokens == 2) { // we do not have any indices - // discard useless token - tok.nextToken(); - indices = new int[0]; - probs = new float[0]; - } else { - indices = new int[(numTokens - 1) / 2]; - // same length - probs = new float[indices.length]; - - while (index * 2 < numTokens - 1) { - String token = tok.nextToken(); - if (index == 0) { - token = token.substring(4); - } else { - token = token.substring(1); - } - // System.out.println("int-token: "+token); - indices[index] = Integer.parseInt(token); - - token = tok.nextToken(); - int lastIndex = token.length() - 1; - if ((index * 2) == (numTokens - 3)) { - token = token.substring(0, lastIndex - 1); - if (token.equals("inf")) { - probs[index] = 10000; - index++; - continue; - } - if (token.equals("nan")) { - probs[index] = -1; - index++; - continue; - } - } else { - token = token.substring(0, lastIndex); - if (token.equals("inf")) { - probs[index] = 1000000; - index++; - continue; - } - if (token.equals("nan")) { - probs[index] = -1; - index++; - continue; - } - } - // System.out.println("float-token: "+token); - probs[index] = Float.parseFloat(token); - index++; - } // end while - - } // end if - - return new LeafNode.StringAndFloatLeafNode(indices, probs); - } - -} diff --git a/marytts-unitselection/src/main/java/marytts/cart/io/WagonCARTWriter.java b/marytts-unitselection/src/main/java/marytts/cart/io/WagonCARTWriter.java deleted file mode 100644 index d46204129b..0000000000 --- a/marytts-unitselection/src/main/java/marytts/cart/io/WagonCARTWriter.java +++ /dev/null @@ -1,379 +0,0 @@ -/** - * Copyright 2006 DFKI GmbH. - * All Rights Reserved. Use is subject to license terms. - * - * This file is part of MARY TTS. - * - * MARY TTS is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - * - */ -package marytts.cart.io; - -import java.io.BufferedOutputStream; -import java.io.DataOutput; -import java.io.DataOutputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.PrintWriter; - -import marytts.cart.CART; -import marytts.cart.DecisionNode; -import marytts.cart.Node; -import marytts.cart.LeafNode.FeatureVectorLeafNode; -import marytts.cart.LeafNode.FloatLeafNode; -import marytts.cart.LeafNode.IntAndFloatArrayLeafNode; -import marytts.cart.LeafNode.IntArrayLeafNode; -import marytts.features.FeatureVector; -import marytts.util.data.MaryHeader; - -/** - * IO functions for CARTs in WagonCART format - * - * @author Anna Hunecke, Marc Schröder, Marcela Charfuelan - */ -public class WagonCARTWriter { - - /** - * Dump the CARTs in the cart map to destinationDir/CARTS.bin - * - * @param cart - * tree - * @param destDir - * the destination directory - */ - public void dumpWagonCART(CART cart, String destFile) throws IOException { - System.out.println("Dumping CART to " + destFile + " ..."); - - // Open the destination file (cart.bin) and output the header - DataOutputStream out = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(destFile))); - // create new CART-header and write it to output file - MaryHeader hdr = new MaryHeader(MaryHeader.CARTS); - hdr.writeTo(out); - - // write number of nodes - out.writeInt(cart.getNumNodes()); - String name = ""; - // dump name and CART - out.writeUTF(name); - // dump CART - dumpBinary(cart, out); - - // finish - out.close(); - System.out.println(" ... done\n"); - } - - /** - * Debug output to a text file - * - * @param cart - * the cart tree - * @param pw - * the print writer of the text file - * @throws IOException - */ - public void toTextOut(CART cart, PrintWriter pw) throws IOException { - try { - toWagonFormat(cart.getRootNode(), null, "", pw); - pw.flush(); - pw.close(); - } catch (IOException ioe) { - IOException newIOE = new IOException("Error dumping CART to standard output"); - newIOE.initCause(ioe); - throw newIOE; - } - } - - /** - * Dumps this CART to the output stream in WagonFormat. - * - * @param os - * the output stream - * @param cart - * the cart tree - * - * @throws IOException - * if an error occurs during output - */ - public void dumpBinary(CART cart, DataOutput os) throws IOException { - try { - toWagonFormat(cart.getRootNode(), (DataOutputStream) os, null, null); - } catch (IOException ioe) { - IOException newIOE = new IOException("Error dumping CART to output stream"); - newIOE.initCause(ioe); - throw newIOE; - } - } - - private void toWagonFormat(Node node, DataOutputStream out, String extension, PrintWriter pw) throws IOException { - - if (node instanceof DecisionNode) - toWagonFormat(((DecisionNode) node), out, extension, pw); - - else if (node instanceof FeatureVectorLeafNode) - toWagonFormat(((FeatureVectorLeafNode) node), out, extension, pw); - - else if (node instanceof FloatLeafNode) - toWagonFormat(((FloatLeafNode) node), out, extension, pw); - - // As StringAndFloatLeafNode is just a convenience wrapper around IntAndFloatArrayLeafNode, - // there is no need to distinguish them in IO. - else if (node instanceof IntAndFloatArrayLeafNode) - toWagonFormat(((IntAndFloatArrayLeafNode) node), out, extension, pw); - - else if (node instanceof IntArrayLeafNode) - toWagonFormat(((IntArrayLeafNode) node), out, extension, pw); - - } - - /** - * Writes the Cart to the given DataOut in Wagon Format - * - * @param out - * the outputStream - * @param extension - * the extension that is added to the last daughter - */ - private void toWagonFormat(DecisionNode node, DataOutputStream out, String extension, PrintWriter pw) throws IOException { - if (out != null) { - // dump to output stream - // two open brackets + definition of node - writeStringToOutput("((" + node.getNodeDefinition() + ")", out); - } else { - // dump to Standard out - // two open brackets + definition of node - // System.out.println("(("+getNodeDefinition()); - } - if (pw != null) { - // dump to print writer - // two open brackets + definition of node - pw.println("((" + node.getNodeDefinition() + ")"); - } - // add the daughters - for (int i = 0; i < node.getNumberOfDaugthers(); i++) { - - if (node.getDaughter(i) == null) { - String nullDaughter = ""; - - if (i + 1 != node.getNumberOfDaugthers()) { - nullDaughter = "((() 0))"; - - } else { - // extension must be added to last daughter - if (extension != null) { - nullDaughter = "((() 0)))" + extension; - - } else { - // we are in the root node, add a closing bracket - nullDaughter = "((() 0)))"; - } - } - - if (out != null) { - // dump to output stream - writeStringToOutput(nullDaughter, out); - } else { - // dump to Standard out - // System.out.println(nullDaughter); - } - if (pw != null) { - pw.println(" " + nullDaughter); - } - } else { - if (i + 1 != node.getNumberOfDaugthers()) { - - toWagonFormat(node.getDaughter(i), out, "", pw); - // daughters[i].toWagonFormat(out, "", pw); - } else { - - // extension must be added to last daughter - if (extension != null) { - toWagonFormat(node.getDaughter(i), out, ")" + extension, pw); - // daughters[i].toWagonFormat(out, ")" + extension, pw); - } else { - // we are in the root node, add a closing bracket - toWagonFormat(node.getDaughter(i), out, ")", pw); - // daughters[i].toWagonFormat(out, ")", pw); - } - } - } - } - } - - /** - * Writes the Cart to the given DataOut in Wagon Format - * - * @param out - * the outputStream - * @param extension - * the extension that is added to the last daughter - */ - private void toWagonFormat(FeatureVectorLeafNode node, DataOutputStream out, String extension, PrintWriter pw) - throws IOException { - StringBuilder sb = new StringBuilder(); - FeatureVector fv[] = node.getFeatureVectors(); - - // open three brackets - sb.append("((("); - // make sure that we have a feature vector array - // this is done when calling getFeatureVectors(). - // if (growable && - // (featureVectors == null - // || featureVectors.length == 0)){ - // featureVectors = (FeatureVector[]) - // featureVectorList.toArray( - // new FeatureVector[featureVectorList.size()]); - // } - // for each index, write the index and then a pseudo float - for (int i = 0; i < fv.length; i++) { - sb.append("(" + fv[i].getUnitIndex() + " 0)"); - if (i + 1 != fv.length) { - sb.append(" "); - } - } - // write the ending - sb.append(") 0))" + extension); - // dump the whole stuff - if (out != null) { - // write to output stream - - writeStringToOutput(sb.toString(), out); - } else { - // write to Standard out - // System.out.println(sb.toString()); - } - if (pw != null) { - // dump to printwriter - pw.println(sb.toString()); - } - } - - /** - * Writes the Cart to the given DataOut in Wagon Format - * - * @param out - * the outputStream - * @param extension - * the extension that is added to the last daughter - */ - private void toWagonFormat(FloatLeafNode node, DataOutputStream out, String extension, PrintWriter pw) throws IOException { - String s = "((" + node.getStDeviation() // stddev - + " " + node.getMean() // mean - + "))"; - // dump the whole stuff - if (out != null) { - // write to output stream - - writeStringToOutput(s, out); - } else { - // write to Standard out - // System.out.println(sb.toString()); - } - if (pw != null) { - // dump to printwriter - pw.println(s); - } - } - - /** - * Writes the Cart to the given DataOut in Wagon Format - * - * @param out - * the outputStream - * @param extension - * the extension that is added to the last daughter - */ - private void toWagonFormat(IntAndFloatArrayLeafNode node, DataOutputStream out, String extension, PrintWriter pw) - throws IOException { - StringBuilder sb = new StringBuilder(); - int data[] = node.getIntData(); - float floats[] = node.getFloatData(); - - // open three brackets - sb.append("((("); - // for each index, write the index and then its float - for (int i = 0; i < data.length; i++) { - sb.append("(" + data[i] + " " + floats[i] + ")"); - if (i + 1 != data.length) { - sb.append(" "); - } - } - // write the ending - sb.append(") 0))" + extension); - // dump the whole stuff - if (out != null) { - // write to output stream - - writeStringToOutput(sb.toString(), out); - } else { - // write to Standard out - // System.out.println(sb.toString()); - } - if (pw != null) { - // dump to printwriter - pw.println(sb.toString()); - } - } - - /** - * Writes the Cart to the given DataOut in Wagon Format - * - * @param out - * the outputStream - * @param extension - * the extension that is added to the last daughter - */ - private void toWagonFormat(IntArrayLeafNode node, DataOutputStream out, String extension, PrintWriter pw) throws IOException { - StringBuilder sb = new StringBuilder(); - int data[] = node.getIntData(); - - // open three brackets - sb.append("((("); - // for each index, write the index and then a pseudo float - for (int i = 0; i < data.length; i++) { - sb.append("(" + data[i] + " 0)"); - if (i + 1 != data.length) { - sb.append(" "); - } - } - // write the ending - sb.append(") 0))" + extension); - // dump the whole stuff - if (out != null) { - // write to output stream - - writeStringToOutput(sb.toString(), out); - } else { - // write to Standard out - // System.out.println(sb.toString()); - } - if (pw != null) { - // dump to printwriter - pw.println(sb.toString()); - } - } - - /** - * Write the given String to the given data output (Replacement for writeUTF) - * - * @param str - * the String - * @param out - * the data output - */ - private static void writeStringToOutput(String str, DataOutput out) throws IOException { - out.writeInt(str.length()); - out.writeChars(str); - } - -} From a71db751b72ac16c695d9fd36531b0101f746a80 Mon Sep 17 00:00:00 2001 From: Tristan Hamilton Date: Fri, 6 Feb 2015 14:53:40 +0100 Subject: [PATCH 21/31] clean reference to FeatureFileReader --- .../src/main/java/marytts/modules/PolynomialF0Modeller.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/marytts-runtime/src/main/java/marytts/modules/PolynomialF0Modeller.java b/marytts-runtime/src/main/java/marytts/modules/PolynomialF0Modeller.java index cea5f6b912..958f3bcf2f 100644 --- a/marytts-runtime/src/main/java/marytts/modules/PolynomialF0Modeller.java +++ b/marytts-runtime/src/main/java/marytts/modules/PolynomialF0Modeller.java @@ -33,7 +33,7 @@ import marytts.modules.phonemiser.AllophoneSet; import marytts.modules.synthesis.Voice; import marytts.server.MaryProperties; -import marytts.unitselection.data.FeatureFileReader; +import marytts.features.FeatureFileReader; import marytts.features.Target; import marytts.util.MaryRuntimeUtils; import marytts.util.MaryUtils; From d3f6a35c362f158f9759171266cf704be143abcb Mon Sep 17 00:00:00 2001 From: Tristan Hamilton Date: Fri, 6 Feb 2015 16:20:18 +0100 Subject: [PATCH 22/31] CARTF0Modeller sent to purgatory --- .../src/main/java/marytts/modules/CARTF0Modeller.java | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {marytts-runtime => marytts-purgatory}/src/main/java/marytts/modules/CARTF0Modeller.java (100%) diff --git a/marytts-runtime/src/main/java/marytts/modules/CARTF0Modeller.java b/marytts-purgatory/src/main/java/marytts/modules/CARTF0Modeller.java similarity index 100% rename from marytts-runtime/src/main/java/marytts/modules/CARTF0Modeller.java rename to marytts-purgatory/src/main/java/marytts/modules/CARTF0Modeller.java From da9b24bd6576990843ee9406db017bb497ab3d47 Mon Sep 17 00:00:00 2001 From: Tristan Hamilton Date: Fri, 6 Feb 2015 17:23:54 +0100 Subject: [PATCH 23/31] moved domain and example text from unitselectionvoice to voice --- .../java/marytts/modules/synthesis/Voice.java | 51 ++++++++++++++++++- .../unitselection/UnitSelectionVoice.java | 43 ---------------- 2 files changed, 50 insertions(+), 44 deletions(-) diff --git a/marytts-runtime/src/main/java/marytts/modules/synthesis/Voice.java b/marytts-runtime/src/main/java/marytts/modules/synthesis/Voice.java index 37b26c1d73..a9a08f8452 100644 --- a/marytts-runtime/src/main/java/marytts/modules/synthesis/Voice.java +++ b/marytts-runtime/src/main/java/marytts/modules/synthesis/Voice.java @@ -145,6 +145,8 @@ public int compare(Voice v1, Voice v2) { private AudioFormat dbAudioFormat = null; private WaveformSynthesizer synthesizer; private Gender gender; + protected String domain; + protected String exampleText; private int wantToBeDefault; private AllophoneSet allophoneSet; String preferredModulesClasses; @@ -191,7 +193,20 @@ public Voice(String name, WaveformSynthesizer synthesizer) throws MaryConfigurat false); this.gender = new Gender(MaryProperties.needProperty("voice." + voiceName + ".gender")); - + + //taken from UnitSeletionVoice + String header = "voice." + name; + this.domain = MaryProperties.getProperty(header + ".domain"); + InputStream exampleTextStream = null; + try { + exampleTextStream = MaryProperties.getStream(header + ".exampleTextFile"); + if (exampleTextStream != null) { + readExampleText(exampleTextStream); + } + } catch (Exception ex) { + throw new MaryConfigurationException("No .exampleTextFile found", ex); + } + try { init(); } catch (Exception n) { @@ -535,6 +550,15 @@ public AudioInputStream synthesize(List tokensAndBoundaries, String out public Lexicon getLexicon() { return lexicon; } + + /** + * Gets the domain of this voice + * + * @return the domain + */ + public String getDomain() { + return domain; + } public DirectedGraph getDurationGraph() { return durationGraph; @@ -930,6 +954,31 @@ private static Lexicon getLexicon(String lexiconClass, String lexiconName) { lexicons.put(lexiconClass + lexiconName, lexicon); return lexicon; } + + public void readExampleText(InputStream in) throws IOException { + BufferedReader reader = new BufferedReader(new InputStreamReader(in, "UTF-8")); + StringBuilder sb = new StringBuilder(); + String line = reader.readLine(); + while (line != null) { + if (!line.startsWith("***")) { + sb.append(line + "\n"); + } + line = reader.readLine(); + } + this.exampleText = sb.toString(); + } + + public String getExampleText() { + if (exampleText == null) { + return ""; + } else { + return exampleText; + } + } + + public boolean isUnitSelection() { + return MaryRuntimeUtils.getVoicesList("unitselection").contains(this.getName()); + } public static class Gender { String name; diff --git a/marytts-unitselection/src/main/java/marytts/unitselection/UnitSelectionVoice.java b/marytts-unitselection/src/main/java/marytts/unitselection/UnitSelectionVoice.java index c11d484424..2cfb225bd4 100644 --- a/marytts-unitselection/src/main/java/marytts/unitselection/UnitSelectionVoice.java +++ b/marytts-unitselection/src/main/java/marytts/unitselection/UnitSelectionVoice.java @@ -72,10 +72,8 @@ public class UnitSelectionVoice extends Voice { protected UnitSelector unitSelector; protected UnitConcatenator concatenator; protected UnitConcatenator modificationConcatenator; - protected String domain; protected String name; protected CART[] f0Carts; - protected String exampleText; public UnitSelectionVoice(String name, WaveformSynthesizer synthesizer) throws MaryConfigurationException { super(name, synthesizer); @@ -84,17 +82,6 @@ public UnitSelectionVoice(String name, WaveformSynthesizer synthesizer) throws M this.name = name; String header = "voice." + name; - domain = MaryProperties.needProperty(header + ".domain"); - InputStream exampleTextStream = null; - if (!domain.equals("general")) { // limited domain voices must have example text; - exampleTextStream = MaryProperties.needStream(header + ".exampleTextFile"); - } else { // general domain voices can have example text: - exampleTextStream = MaryProperties.getStream(header + ".exampleTextFile"); - } - if (exampleTextStream != null) { - readExampleText(exampleTextStream); - } - FeatureProcessorManager featProcManager = FeatureRegistry.getFeatureProcessorManager(this); if (featProcManager == null) featProcManager = FeatureRegistry.getFeatureProcessorManager(getLocale()); @@ -282,36 +269,6 @@ public UnitConcatenator getModificationConcatenator() { return modificationConcatenator; } - /** - * Gets the domain of this voice - * - * @return the domain - */ - public String getDomain() { - return domain; - } - - public String getExampleText() { - if (exampleText == null) { - return ""; - } else { - return exampleText; - } - } - - public void readExampleText(InputStream in) throws IOException { - BufferedReader reader = new BufferedReader(new InputStreamReader(in, "UTF-8")); - StringBuilder sb = new StringBuilder(); - String line = reader.readLine(); - while (line != null) { - if (!line.startsWith("***")) { - sb.append(line + "\n"); - } - line = reader.readLine(); - } - exampleText = sb.toString(); - } - public CART[] getF0Trees() { return f0Carts; } From c0a80925d4308cfc8482dc7596d2090d9ad90561 Mon Sep 17 00:00:00 2001 From: Tristan Hamilton Date: Fri, 6 Feb 2015 17:32:50 +0100 Subject: [PATCH 24/31] previous commit added isUnitSelection to Voice, this allows for UnitSelectionVoice to be completely omitted from runtime --- .../java/marytts/modules/SoPF0Modeller.java | 4 ++-- .../java/marytts/modules/synthesis/Voice.java | 6 ++--- .../main/java/marytts/server/MaryServer.java | 7 +++--- .../java/marytts/util/MaryRuntimeUtils.java | 23 ++++++++++++------- 4 files changed, 23 insertions(+), 17 deletions(-) diff --git a/marytts-runtime/src/main/java/marytts/modules/SoPF0Modeller.java b/marytts-runtime/src/main/java/marytts/modules/SoPF0Modeller.java index 65e76fe5e3..2476466c51 100644 --- a/marytts-runtime/src/main/java/marytts/modules/SoPF0Modeller.java +++ b/marytts-runtime/src/main/java/marytts/modules/SoPF0Modeller.java @@ -37,7 +37,6 @@ import marytts.modules.phonemiser.AllophoneSet; import marytts.modules.synthesis.Voice; import marytts.server.MaryProperties; -import marytts.unitselection.UnitSelectionVoice; import marytts.features.Target; import marytts.util.MaryRuntimeUtils; import marytts.util.MaryUtils; @@ -183,7 +182,8 @@ public MaryData process(MaryData d) throws Exception { SoP currentMidSop = midSop; SoP currentRightSop = rightSop; TargetFeatureComputer currentFeatureComputer = featureComputer; - if (maryVoice instanceof UnitSelectionVoice) { + //if (maryVoice instanceof UnitSelectionVoice) { + if (maryVoice.isUnitSelection()) { if (voiceFeatDef != null) { currentFeatureComputer = new TargetFeatureComputer(featureProcessorManager, voiceFeatDef.getFeatureNames()); logger.debug("Using voice feature definition"); diff --git a/marytts-runtime/src/main/java/marytts/modules/synthesis/Voice.java b/marytts-runtime/src/main/java/marytts/modules/synthesis/Voice.java index a9a08f8452..37175998fe 100644 --- a/marytts-runtime/src/main/java/marytts/modules/synthesis/Voice.java +++ b/marytts-runtime/src/main/java/marytts/modules/synthesis/Voice.java @@ -69,7 +69,6 @@ import marytts.modules.phonemiser.Allophone; import marytts.modules.phonemiser.AllophoneSet; import marytts.server.MaryProperties; -import marytts.unitselection.UnitSelectionVoice; import marytts.util.MaryRuntimeUtils; import marytts.util.MaryUtils; @@ -709,7 +708,7 @@ public static Collection getAvailableVoices() { public static List getFilteredVoice(String queryVariable, String queryValue, Collection initialList) { List filteredList = new ArrayList(); - + switch (queryVariable.toLowerCase()) { case "locale": Locale queryLocale = MaryUtils.string2locale(queryValue); @@ -739,7 +738,8 @@ public static List getFilteredVoice(String queryVariable, String queryVal case "type": for (Voice currentVoice : initialList) { boolean isQualified = false; - if (queryValue.toLowerCase().contains("unit") && currentVoice instanceof UnitSelectionVoice) { + //if (queryValue.toLowerCase().contains("unit") && currentVoice instanceof UnitSelectionVoice) { + if (queryValue.toLowerCase().contains("unit") && currentVoice.isUnitSelection()) { isQualified = true; } else if (queryValue.toLowerCase().contains("hmm") && currentVoice instanceof HMMVoice) { isQualified = true; diff --git a/marytts-runtime/src/main/java/marytts/server/MaryServer.java b/marytts-runtime/src/main/java/marytts/server/MaryServer.java index 196df602b1..464a3a01c9 100644 --- a/marytts-runtime/src/main/java/marytts/server/MaryServer.java +++ b/marytts-runtime/src/main/java/marytts/server/MaryServer.java @@ -52,7 +52,6 @@ import marytts.signalproc.effects.AudioEffect; import marytts.signalproc.effects.AudioEffects; import marytts.signalproc.effects.BaseAudioEffect; -import marytts.unitselection.UnitSelectionVoice; import marytts.util.MaryRuntimeUtils; import marytts.util.MaryUtils; import marytts.util.data.audio.MaryAudioUtils; @@ -575,9 +574,9 @@ private boolean listLocales() { private boolean listVoices() { // list all known voices for (Voice v : Voice.getAvailableVoices()) { - if (v instanceof UnitSelectionVoice) { + if (v.isUnitSelection()) { clientOut.println(v.getName() + " " + v.getLocale() + " " + v.gender().toString() + " " + "unitselection" - + " " + ((UnitSelectionVoice) v).getDomain()); + + " " + v.getDomain()); } else if (v instanceof HMMVoice) { clientOut.println(v.getName() + " " + v.getLocale() + " " + v.gender().toString() + " " + "hmm"); } else { @@ -624,7 +623,7 @@ private boolean voiceExampleText(String inputLine) { try { String voiceName = st.nextToken(); Voice v = Voice.getVoice(voiceName); - String text = ((marytts.unitselection.UnitSelectionVoice) v).getExampleText(); + String text = v.getExampleText(); if (text != null) { clientOut.println(text); } diff --git a/marytts-runtime/src/main/java/marytts/util/MaryRuntimeUtils.java b/marytts-runtime/src/main/java/marytts/util/MaryRuntimeUtils.java index 27898a70e3..387876d5e5 100644 --- a/marytts-runtime/src/main/java/marytts/util/MaryRuntimeUtils.java +++ b/marytts-runtime/src/main/java/marytts/util/MaryRuntimeUtils.java @@ -54,7 +54,6 @@ import marytts.server.MaryProperties; import marytts.signalproc.effects.AudioEffect; import marytts.signalproc.effects.AudioEffects; -import marytts.unitselection.UnitSelectionVoice; import marytts.util.data.audio.AudioDestination; import marytts.util.data.audio.MaryAudioUtils; import marytts.util.dom.MaryDomUtils; @@ -391,9 +390,9 @@ public static void sortVoicesList(List voices) { Collections.sort(voices, new Comparator() { @Override public int compare(Voice v1, Voice v2) { - return (v1.getLocale().toString() + " " + v1.gender().toString() + " " + v1.getName() + " " + (v1 instanceof UnitSelectionVoice ? "u" + return (v1.getLocale().toString() + " " + v1.gender().toString() + " " + v1.getName() + " " + (v1.isUnitSelection() ? "u" : "h")).compareToIgnoreCase(v2.getLocale().toString() + " " + v2.gender().toString() + " " + v2.getName() - + " " + (v2 instanceof UnitSelectionVoice ? "u" : "h")); + + " " + (v2.isUnitSelection() ? "u" : "h")); } }); } @@ -404,12 +403,12 @@ public static String formJSONFromVoices(Collection voices) { for (Iterator it = voices.iterator(); it.hasNext();) { JsonObject currentVoice = new JsonObject(); Voice v = (Voice) it.next(); - if (v instanceof UnitSelectionVoice) { + if (v.isUnitSelection()) { currentVoice.addProperty("name", v.getName()); currentVoice.addProperty("locale", v.getLocale().toString()); currentVoice.addProperty("gender", v.gender().toString()); currentVoice.addProperty("type", "unitselection"); - currentVoice.addProperty("domain", ((UnitSelectionVoice) v).getDomain()); + currentVoice.addProperty("domain", v.getDomain()); } else if (v instanceof HMMVoice) { currentVoice.addProperty("name", v.getName()); currentVoice.addProperty("locale", v.getLocale().toString()); @@ -470,9 +469,7 @@ public static Vector getDefaultVoiceExampleTexts() { public static String getVoiceExampleText(String voiceName) { Voice v = Voice.getVoice(voiceName); - if (v instanceof marytts.unitselection.UnitSelectionVoice) - return ((marytts.unitselection.UnitSelectionVoice) v).getExampleText(); - return ""; + return v.getExampleText(); } /** @@ -537,4 +534,14 @@ public static String isHmmAudioEffect(String effectName) { } return effect.isHMMEffect() ? "yes" : "no"; } + /** + * returns a list of strings of voice names, depending on voice type wanted + * parameter passed-> a string such as "unitselection" or "hmm" + * @param voicetype + * @return VoiceNames + */ + public static List getVoicesList(String voicetype) { + List VoiceNames = MaryProperties.getList(voicetype + ".voices.list"); + return VoiceNames; + } } \ No newline at end of file From 6527c83c1b00800d1215a5776c22e0cbd0035dd9 Mon Sep 17 00:00:00 2001 From: Tristan Hamilton Date: Mon, 9 Feb 2015 13:10:56 +0100 Subject: [PATCH 25/31] send Vocalizations to purgatory --- .../VocalizationFFRTargetCostFunction.java | 262 +++++++ .../FDPSOLASynthesisTechnology.java | 260 +++++++ .../vocalizations/HNMFeatureFileReader.java | 142 ++++ .../vocalizations/HNMSynthesisTechnology.java | 298 ++++++++ .../vocalizations/KMeansClusterer.java | 188 ++++++ .../vocalizations/MLSAFeatureFileReader.java | 273 ++++++++ .../MLSASynthesisTechnology.java | 283 ++++++++ .../vocalizations/SourceTargetPair.java | 65 ++ .../vocalizations/VocalizationCandidate.java | 58 ++ .../VocalizationFeatureFileReader.java | 89 +++ .../VocalizationIntonationReader.java | 200 ++++++ .../vocalizations/VocalizationSelector.java | 639 ++++++++++++++++++ .../VocalizationSynthesisTechnology.java | 77 +++ .../VocalizationSynthesizer.java | 305 +++++++++ .../vocalizations/VocalizationUnit.java | 59 ++ .../VocalizationUnitFileReader.java | 240 +++++++ 16 files changed, 3438 insertions(+) create mode 100644 marytts-purgatory/src/main/java/marytts/unitselection/select/VocalizationFFRTargetCostFunction.java create mode 100644 marytts-purgatory/src/main/java/marytts/vocalizations/FDPSOLASynthesisTechnology.java create mode 100644 marytts-purgatory/src/main/java/marytts/vocalizations/HNMFeatureFileReader.java create mode 100644 marytts-purgatory/src/main/java/marytts/vocalizations/HNMSynthesisTechnology.java create mode 100644 marytts-purgatory/src/main/java/marytts/vocalizations/KMeansClusterer.java create mode 100644 marytts-purgatory/src/main/java/marytts/vocalizations/MLSAFeatureFileReader.java create mode 100644 marytts-purgatory/src/main/java/marytts/vocalizations/MLSASynthesisTechnology.java create mode 100644 marytts-purgatory/src/main/java/marytts/vocalizations/SourceTargetPair.java create mode 100644 marytts-purgatory/src/main/java/marytts/vocalizations/VocalizationCandidate.java create mode 100644 marytts-purgatory/src/main/java/marytts/vocalizations/VocalizationFeatureFileReader.java create mode 100644 marytts-purgatory/src/main/java/marytts/vocalizations/VocalizationIntonationReader.java create mode 100644 marytts-purgatory/src/main/java/marytts/vocalizations/VocalizationSelector.java create mode 100644 marytts-purgatory/src/main/java/marytts/vocalizations/VocalizationSynthesisTechnology.java create mode 100644 marytts-purgatory/src/main/java/marytts/vocalizations/VocalizationSynthesizer.java create mode 100644 marytts-purgatory/src/main/java/marytts/vocalizations/VocalizationUnit.java create mode 100644 marytts-purgatory/src/main/java/marytts/vocalizations/VocalizationUnitFileReader.java diff --git a/marytts-purgatory/src/main/java/marytts/unitselection/select/VocalizationFFRTargetCostFunction.java b/marytts-purgatory/src/main/java/marytts/unitselection/select/VocalizationFFRTargetCostFunction.java new file mode 100644 index 0000000000..c6a0163eb2 --- /dev/null +++ b/marytts-purgatory/src/main/java/marytts/unitselection/select/VocalizationFFRTargetCostFunction.java @@ -0,0 +1,262 @@ +/** + * Copyright 2006 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.unitselection.select; + +import marytts.features.FeatureDefinition; +import marytts.features.FeatureVector; +import marytts.unitselection.data.Unit; +import marytts.unitselection.weightingfunctions.WeightFunc; +import marytts.unitselection.weightingfunctions.WeightFunctionManager; +import marytts.vocalizations.VocalizationFeatureFileReader; + +/** + * FFRTargetCostFunction for vocalization selection + * + * @author sathish pammi + * + */ +public class VocalizationFFRTargetCostFunction extends FFRTargetCostFunction { + + private int MEANING_RATING_RANGE = 5; // the range of meaning rating scale + + public VocalizationFFRTargetCostFunction(VocalizationFeatureFileReader ffr) { + this(ffr, ffr.getFeatureDefinition()); + } + + public VocalizationFFRTargetCostFunction(VocalizationFeatureFileReader ffr, FeatureDefinition fDef) { + load(ffr, fDef); + } + + /** + * load feature file reader and feature definition for a cost function + * + * @param ffr + * feature file reader + * @param fDef + * feature definition + */ + private void load(VocalizationFeatureFileReader ffr, FeatureDefinition fDef) { + this.featureVectors = ffr.featureVectorMapping(fDef); + this.featureDefinition = fDef; + + weightFunction = new WeightFunc[featureDefinition.getNumberOfContinuousFeatures()]; + WeightFunctionManager wfm = new WeightFunctionManager(); + int nDiscreteFeatures = featureDefinition.getNumberOfByteFeatures() + featureDefinition.getNumberOfShortFeatures(); + for (int i = 0; i < weightFunction.length; i++) { + String weightFunctionName = featureDefinition.getWeightFunctionName(nDiscreteFeatures + i); + if ("".equals(weightFunctionName)) + weightFunction[i] = wfm.getWeightFunction("linear"); + else + weightFunction[i] = wfm.getWeightFunction(weightFunctionName); + } + + rememberWhichWeightsAreNonZero(); + } + + /** + * Compute the goodness-of-fit of a given unit for a given target + * + * @param target + * target unit + * @param unit + * candidate unit + * @param weights + * FeatureDefinition + * @param weightFunctions + * array of WeightFunctions + * @return a non-negative number; smaller values mean better fit, i.e. smaller cost. + * @throws IllegalArgumentException + * if featureName not available in featureDefinition + */ + protected double cost(Target target, Unit unit, FeatureDefinition weights, WeightFunc[] weightFunctions) { + nCostComputations++; // for debug + FeatureVector targetFeatures = target.getFeatureVector(); + assert targetFeatures != null : "Target " + target + " does not have pre-computed feature vector"; + FeatureVector unitFeatures = featureVectors[unit.index]; + int nBytes = targetFeatures.byteValuedDiscreteFeatures.length; + int nShorts = targetFeatures.shortValuedDiscreteFeatures.length; + int nFloats = targetFeatures.continuousFeatures.length; + assert nBytes == unitFeatures.byteValuedDiscreteFeatures.length; + assert nShorts == unitFeatures.shortValuedDiscreteFeatures.length; + assert nFloats == unitFeatures.continuousFeatures.length; + + float[] weightVector = weights.getFeatureWeights(); + // Now the actual computation + double cost = 0; + // byte-valued features: + if (nBytes > 0) { + for (int i = 0; i < nBytes; i++) { + if (weightsNonZero[i]) { + float weight = weightVector[i]; + if (featureDefinition.hasSimilarityMatrix(i)) { + byte targetFeatValueIndex = targetFeatures.byteValuedDiscreteFeatures[i]; + byte unitFeatValueIndex = unitFeatures.byteValuedDiscreteFeatures[i]; + float similarity = featureDefinition.getSimilarity(i, unitFeatValueIndex, targetFeatValueIndex); + cost += similarity * weight; + if (debugShowCostGraph) + cumulWeightedCosts[i] += similarity * weight; + } else if (targetFeatures.byteValuedDiscreteFeatures[i] != unitFeatures.byteValuedDiscreteFeatures[i]) { + cost += weight; + if (debugShowCostGraph) + cumulWeightedCosts[i] += weight; + } + } + } + } + // short-valued features: + if (nShorts > 0) { + for (int i = nBytes, n = nBytes + nShorts; i < n; i++) { + if (weightsNonZero[i]) { + float weight = weightVector[i]; + // if (targetFeatures.getShortFeature(i) != unitFeatures.getShortFeature(i)) { + if (targetFeatures.shortValuedDiscreteFeatures[i - nBytes] != unitFeatures.shortValuedDiscreteFeatures[i + - nBytes]) { + cost += weight; + if (debugShowCostGraph) + cumulWeightedCosts[i] += weight; + } + } + } + } + // continuous features: + if (nFloats > 0) { + int nDiscrete = nBytes + nShorts; + for (int i = nDiscrete, n = nDiscrete + nFloats; i < n; i++) { + if (weightsNonZero[i]) { + float weight = weightVector[i]; + // float a = targetFeatures.getContinuousFeature(i); + float a = targetFeatures.continuousFeatures[i - nDiscrete]; + // float b = unitFeatures.getContinuousFeature(i); + float b = unitFeatures.continuousFeatures[i - nDiscrete]; + // if (!Float.isNaN(a) && !Float.isNaN(b)) { + // Implementation of isNaN() is: (v != v). + if (!(a != a)) { + + double myCost; + if (!(b != b)) { + myCost = weightFunctions[i - nDiscrete].cost(a, b); + } else { + myCost = this.MEANING_RATING_RANGE; + } + + cost += weight * myCost; + if (debugShowCostGraph) { + cumulWeightedCosts[i] += weight * myCost; + } + } // and if it is NaN, simply compute no cost + } + } + } + return cost; + } + + /** + * Compute the goodness-of-fit between given unit and given target for a given feature + * + * @param target + * target unit + * @param unit + * candidate unit + * @param featureName + * feature name + * @param weights + * FeatureDefinition + * @param weightFunctions + * array of WeightFunctions + * @return a non-negative number; smaller values mean better fit, i.e. smaller cost. + * @throws IllegalArgumentException + * if featureName not available in featureDefinition + */ + protected double featureCost(Target target, Unit unit, String featureName, FeatureDefinition weights, + WeightFunc[] weightFunctions) { + if (!this.featureDefinition.hasFeature(featureName)) { + throw new IllegalArgumentException("this feature does not exists in feature definition"); + } + + FeatureVector targetFeatures = target.getFeatureVector(); + assert targetFeatures != null : "Target " + target + " does not have pre-computed feature vector"; + FeatureVector unitFeatures = featureVectors[unit.index]; + int nBytes = targetFeatures.byteValuedDiscreteFeatures.length; + int nShorts = targetFeatures.shortValuedDiscreteFeatures.length; + int nFloats = targetFeatures.continuousFeatures.length; + assert nBytes == unitFeatures.byteValuedDiscreteFeatures.length; + assert nShorts == unitFeatures.shortValuedDiscreteFeatures.length; + assert nFloats == unitFeatures.continuousFeatures.length; + + int featureIndex = this.featureDefinition.getFeatureIndex(featureName); + float[] weightVector = weights.getFeatureWeights(); + double cost = 0; + + if (featureIndex < nBytes) { + if (weightsNonZero[featureIndex]) { + float weight = weightVector[featureIndex]; + if (featureDefinition.hasSimilarityMatrix(featureIndex)) { + byte targetFeatValueIndex = targetFeatures.byteValuedDiscreteFeatures[featureIndex]; + byte unitFeatValueIndex = unitFeatures.byteValuedDiscreteFeatures[featureIndex]; + float similarity = featureDefinition.getSimilarity(featureIndex, unitFeatValueIndex, targetFeatValueIndex); + cost = similarity * weight; + if (debugShowCostGraph) + cumulWeightedCosts[featureIndex] += similarity * weight; + } else if (targetFeatures.byteValuedDiscreteFeatures[featureIndex] != unitFeatures.byteValuedDiscreteFeatures[featureIndex]) { + cost = weight; + if (debugShowCostGraph) + cumulWeightedCosts[featureIndex] += weight; + } + } + } else if (featureIndex < nShorts + nBytes) { + if (weightsNonZero[featureIndex]) { + float weight = weightVector[featureIndex]; + // if (targetFeatures.getShortFeature(i) != unitFeatures.getShortFeature(i)) { + if (targetFeatures.shortValuedDiscreteFeatures[featureIndex - nBytes] != unitFeatures.shortValuedDiscreteFeatures[featureIndex + - nBytes]) { + cost = weight; + if (debugShowCostGraph) + cumulWeightedCosts[featureIndex] += weight; + } + } + } else { + int nDiscrete = nBytes + nShorts; + if (weightsNonZero[featureIndex]) { + float weight = weightVector[featureIndex]; + // float a = targetFeatures.getContinuousFeature(i); + float a = targetFeatures.continuousFeatures[featureIndex - nDiscrete]; + // float b = unitFeatures.getContinuousFeature(i); + float b = unitFeatures.continuousFeatures[featureIndex - nDiscrete]; + // if (!Float.isNaN(a) && !Float.isNaN(b)) { + // Implementation of isNaN() is: (v != v). + if (!(a != a)) { + + double myCost; + if (!(b != b)) { + myCost = weightFunctions[featureIndex - nDiscrete].cost(a, b); + } else { + myCost = this.MEANING_RATING_RANGE; + } + + cost = weight * myCost; + if (debugShowCostGraph) { + cumulWeightedCosts[featureIndex] += weight * myCost; + } + } // and if it is NaN, simply compute no cost + } + } + return cost; + } +} diff --git a/marytts-purgatory/src/main/java/marytts/vocalizations/FDPSOLASynthesisTechnology.java b/marytts-purgatory/src/main/java/marytts/vocalizations/FDPSOLASynthesisTechnology.java new file mode 100644 index 0000000000..2d91fba6cd --- /dev/null +++ b/marytts-purgatory/src/main/java/marytts/vocalizations/FDPSOLASynthesisTechnology.java @@ -0,0 +1,260 @@ +/** + * Copyright 2000-2006 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.vocalizations; + +import java.io.IOException; +import java.util.Arrays; +import java.util.LinkedList; + +import javax.sound.sampled.AudioFileFormat; +import javax.sound.sampled.AudioFormat; +import javax.sound.sampled.AudioInputStream; + +import marytts.exceptions.MaryConfigurationException; +import marytts.exceptions.SynthesisException; +import marytts.signalproc.process.FDPSOLAProcessor; +import marytts.unitselection.data.TimelineReader; +import marytts.util.data.BufferedDoubleDataSource; +import marytts.util.data.Datagram; +import marytts.util.data.DatagramDoubleDataSource; +import marytts.util.data.DoubleDataSource; +import marytts.util.data.audio.DDSAudioInputStream; +import marytts.util.math.MathUtils; +import marytts.util.math.Polynomial; + +/** + * FDPSOLA Synthesis technology to synthesize vocalizations + * + * @author Sathish Pammi + */ + +public class FDPSOLASynthesisTechnology extends VocalizationSynthesisTechnology { + + protected VocalizationIntonationReader vIntonationReader; + protected TimelineReader audioTimeline; + protected VocalizationUnitFileReader unitFileReader; + protected boolean f0ContourImposeSupport; + + public FDPSOLASynthesisTechnology(String waveTimeLineFile, String unitFile, String intonationFeatureFile, + boolean imposeF0Support) throws MaryConfigurationException { + + try { + this.audioTimeline = new TimelineReader(waveTimeLineFile); + this.unitFileReader = new VocalizationUnitFileReader(unitFile); + this.f0ContourImposeSupport = imposeF0Support; + + if (f0ContourImposeSupport) { + this.vIntonationReader = new VocalizationIntonationReader(intonationFeatureFile); + } else { + this.vIntonationReader = null; + } + } catch (IOException e) { + throw new MaryConfigurationException("Can not read data from files " + e); + } + } + + public FDPSOLASynthesisTechnology(TimelineReader audioTimeline, VocalizationUnitFileReader unitFileReader, + HNMFeatureFileReader vHNMFeaturesReader, VocalizationIntonationReader vIntonationReader, boolean imposeF0Support) { + + this.audioTimeline = audioTimeline; + this.unitFileReader = unitFileReader; + this.vIntonationReader = vIntonationReader; + this.f0ContourImposeSupport = imposeF0Support; + + } + + /** + * Synthesize given vocalization (i.e. unit-selection) + * + * @param unitIndex + * unit index + * @param aft + * audio file format + * @return AudioInputStream of synthesized vocalization + * @throws SynthesisException + * if failed to synthesize vocalization + */ + @Override + public AudioInputStream synthesize(int backchannelNumber, AudioFileFormat aft) throws SynthesisException { + + int numberOfBackChannels = unitFileReader.getNumberOfUnits(); + if (backchannelNumber >= numberOfBackChannels) { + throw new IllegalArgumentException("This voice has " + numberOfBackChannels + + " backchannels only. so it doesn't support unit number " + backchannelNumber); + } + + VocalizationUnit bUnit = unitFileReader.getUnit(backchannelNumber); + long start = bUnit.startTime; + int duration = bUnit.duration; + Datagram[] frames = null; + try { + frames = audioTimeline.getDatagrams(start, duration); + } catch (IOException e) { + throw new SynthesisException("Can not read data from timeline file " + e); + } + // Generate audio from frames + LinkedList datagrams = new LinkedList(); + datagrams.addAll(Arrays.asList(frames)); + DoubleDataSource audioSource = new DatagramDoubleDataSource(datagrams); + // audioSource.getAllData(); + return (new DDSAudioInputStream(new BufferedDoubleDataSource(audioSource), aft.getFormat())); + } + + /** + * Re-synthesize given vocalization using FDPSOLA technology + * + * @param unitIndex + * unit index + * @param aft + * audio file format + * @return AudioInputStream of synthesized vocalization + * @throws SynthesisException + * if failed to synthesize vocalization + */ + public AudioInputStream reSynthesize(int backchannelNumber, AudioFileFormat aft) throws SynthesisException { + double[] pScalesArray = { 1.0f }; + double[] tScalesArray = { 1.0f }; + return synthesizeUsingF0Modification(backchannelNumber, pScalesArray, tScalesArray, aft); + } + + /** + * Impose target intonation contour on given vocalization using HNM technology + * + * @param sourceIndex + * unit index of vocalization + * @param targetIndex + * unit index of target intonation + * @param aft + * audio file format + * @return AudioInputStream of synthesized vocalization + * @throws SynthesisException + * if failed to synthesize vocalization + */ + @Override + public AudioInputStream synthesizeUsingImposedF0(int sourceIndex, int targetIndex, AudioFileFormat aft) + throws SynthesisException { + + if (!f0ContourImposeSupport) { + throw new SynthesisException("Mary configuration of this voice doesn't support intonation contour imposition"); + } + + int numberOfUnits = unitFileReader.getNumberOfUnits(); + if (sourceIndex >= numberOfUnits || targetIndex >= numberOfUnits) { + throw new IllegalArgumentException("sourceIndex(" + sourceIndex + ") and targetIndex(" + targetIndex + + ") are should be less than number of available units (" + numberOfUnits + ")"); + } + + if (sourceIndex == targetIndex) { + return reSynthesize(sourceIndex, aft); + } + + double[] sourceF0 = this.vIntonationReader.getContour(sourceIndex); + double[] targetF0coeffs = this.vIntonationReader.getIntonationCoeffs(targetIndex); + double[] sourceF0coeffs = this.vIntonationReader.getIntonationCoeffs(sourceIndex); + + if (targetF0coeffs == null || sourceF0coeffs == null) { + return reSynthesize(sourceIndex, aft); + } + + if (targetF0coeffs.length == 0 || sourceF0coeffs.length == 0) { + return reSynthesize(sourceIndex, aft); + } + + double[] targetF0 = Polynomial.generatePolynomialValues(targetF0coeffs, sourceF0.length, 0, 1); + sourceF0 = Polynomial.generatePolynomialValues(sourceF0coeffs, sourceF0.length, 0, 1); + + assert targetF0.length == sourceF0.length; + double[] tScalesArray = new double[sourceF0.length]; + double[] pScalesArray = new double[sourceF0.length]; + + for (int i = 0; i < targetF0.length; i++) { + pScalesArray[i] = (float) (targetF0[i] / sourceF0[i]); + tScalesArray[i] = (float) (1.0); + } + + return synthesizeUsingF0Modification(sourceIndex, pScalesArray, tScalesArray, aft); + } + + /** + * modify intonation contour using HNM technology + * + * @param backchannelNumber + * unit index of vocalization + * @param pScalesArray + * pitch scales array + * @param tScalesArray + * time scales array + * @param aft + * audio file format + * @return AudioInputStream of synthesized vocalization + * @throws SynthesisException + * if failed to synthesize vocalization + */ + private AudioInputStream synthesizeUsingF0Modification(int backchannelNumber, double[] pScalesArray, double[] tScalesArray, + AudioFileFormat aft) throws SynthesisException { + + if (backchannelNumber > unitFileReader.getNumberOfUnits()) { + throw new IllegalArgumentException("requesting unit should not be more than number of units"); + } + + if (!f0ContourImposeSupport) { + throw new SynthesisException("Mary configuration of this voice doesn't support intonation contour imposition"); + } + + VocalizationUnit bUnit = unitFileReader.getUnit(backchannelNumber); + long start = bUnit.startTime; + int duration = bUnit.duration; + Datagram[] frames = null; + try { + frames = audioTimeline.getDatagrams(start, duration); + } catch (IOException e) { + throw new SynthesisException("cannot get audio frames from timeline file " + e); + } + assert frames != null : "Cannot generate audio from null frames"; + + pScalesArray = MathUtils.arrayResize(pScalesArray, frames.length); + tScalesArray = MathUtils.arrayResize(tScalesArray, frames.length); + + assert tScalesArray.length == pScalesArray.length; + assert frames.length == tScalesArray.length; + + AudioFormat af; + if (aft == null) { // default audio format + float sampleRate = 16000.0F; // 8000,11025,16000,22050,44100 + int sampleSizeInBits = 16; // 8,16 + int channels = 1; // 1,2 + boolean signed = true; // true,false + boolean bigEndian = false; // true,false + af = new AudioFormat(sampleRate, sampleSizeInBits, channels, signed, bigEndian); + } else { + af = aft.getFormat(); + } + + double[] audio_double = (new FDPSOLAProcessor()).processDatagram(frames, null, af, null, pScalesArray, tScalesArray, + false); + /* Normalise the signal before return, this will normalise between 1 and -1 */ + double MaxSample = MathUtils.getAbsMax(audio_double); + for (int i = 0; i < audio_double.length; i++) { + audio_double[i] = 0.3 * (audio_double[i] / MaxSample); + } + return (new DDSAudioInputStream(new BufferedDoubleDataSource(audio_double), af)); + } + +} diff --git a/marytts-purgatory/src/main/java/marytts/vocalizations/HNMFeatureFileReader.java b/marytts-purgatory/src/main/java/marytts/vocalizations/HNMFeatureFileReader.java new file mode 100644 index 0000000000..20b6a3a9d4 --- /dev/null +++ b/marytts-purgatory/src/main/java/marytts/vocalizations/HNMFeatureFileReader.java @@ -0,0 +1,142 @@ +/** + * Portions Copyright 2006 DFKI GmbH. + * Portions Copyright 2001 Sun Microsystems, Inc. + * Portions Copyright 1999-2001 Language Technologies Institute, + * Carnegie Mellon University. + * All Rights Reserved. Use is subject to license terms. + * + * Permission is hereby granted, free of charge, to use and distribute + * this software and its documentation without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of this work, and to + * permit persons to whom this work is furnished to do so, subject to + * the following conditions: + * + * 1. The code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * 2. Any modifications must be clearly marked as such. + * 3. Original authors' names are not deleted. + * 4. The authors' names are not used to endorse or promote products + * derived from this software without specific prior written + * permission. + * + * DFKI GMBH AND THE CONTRIBUTORS TO THIS WORK DISCLAIM ALL WARRANTIES WITH + * REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL DFKI GMBH NOR THE + * CONTRIBUTORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL + * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR + * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS + * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF + * THIS SOFTWARE. + */ +package marytts.vocalizations; + +import java.io.BufferedInputStream; +import java.io.DataInputStream; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; + +import marytts.exceptions.MaryConfigurationException; +import marytts.signalproc.sinusoidal.hntm.analysis.HntmAnalyzerParams; +import marytts.signalproc.sinusoidal.hntm.analysis.HntmSpeechSignal; +import marytts.util.data.MaryHeader; + +/** + * Reads a single file which contains HNM analysis features of vocalizations + * + * @author sathish pammi + * + */ +public class HNMFeatureFileReader { + + private MaryHeader hdr = null; + private HntmSpeechSignal[] hnmSignals; + private int numberOfUnits = 0; + + /** + * Create a feature file reader from the given HNM feature file + * + * @param fileName + * the unit file to read + * @throws IOException + * if a problem occurs while reading + * @throws MaryConfigurationException + * if runtime configuration fails + */ + public HNMFeatureFileReader(String fileName) throws IOException, MaryConfigurationException { + load(fileName); + } + + /** + * Load the given feature file + * + * @param fileName + * the feature file to read + * @throws IOException + * if a problem occurs while reading + * @throws MaryConfigurationException + * if runtime configuration fails + */ + private void load(String fileName) throws IOException, MaryConfigurationException { + // Open the file + DataInputStream dis = null; + + try { + dis = new DataInputStream(new BufferedInputStream(new FileInputStream(fileName))); + } catch (FileNotFoundException e) { + throw new MaryConfigurationException("File [" + fileName + "] was not found."); + } + + // Load the Mary header + hdr = new MaryHeader(dis); + if (hdr.getType() != MaryHeader.LISTENERFEATS) { + throw new MaryConfigurationException("File [" + fileName + "] is not a valid Mary Units file."); + } + + numberOfUnits = dis.readInt(); // Read the number of units + if (numberOfUnits < 0) { + throw new MaryConfigurationException("File [" + fileName + "] has a negative number of units. Aborting."); + } + + hnmSignals = new HntmSpeechSignal[numberOfUnits]; + for (int i = 0; i < numberOfUnits; i++) { + hnmSignals[i] = new HntmSpeechSignal(0, 0, 0); + hnmSignals[i].read(dis, HntmAnalyzerParams.WAVEFORM); + } + + System.out.println(); + } + + /** + * Get the number of units in the file. + * + * @return The number of units. + */ + public int getNumberOfUnits() { + return (numberOfUnits); + } + + /** + * get HntmSpeechSignal for a unit index + * + * @param unitnumber + * unit index number + * @return HntmSpeechSignal hnm analysis feature + * @throws IllegalArgumentException + * if given index number is not less than available units + */ + public HntmSpeechSignal getHntmSpeechSignal(int unitnumber) { + if (unitnumber >= this.numberOfUnits) { + throw new IllegalArgumentException("the given unit index number(" + unitnumber + + ") must be less than number of available units(" + this.numberOfUnits + ")"); + } + return this.hnmSignals[unitnumber]; + } + + public static void main(String[] args) throws Exception { + String fileName = "/home/sathish/Work/phd/voices/mlsa-poppy-listener/vocalizations/files/vocalization_hnm_analysis.mry"; + HNMFeatureFileReader bcUfr = new HNMFeatureFileReader(fileName); + // bcUfr.load(fileName); + } +} diff --git a/marytts-purgatory/src/main/java/marytts/vocalizations/HNMSynthesisTechnology.java b/marytts-purgatory/src/main/java/marytts/vocalizations/HNMSynthesisTechnology.java new file mode 100644 index 0000000000..c22dff689e --- /dev/null +++ b/marytts-purgatory/src/main/java/marytts/vocalizations/HNMSynthesisTechnology.java @@ -0,0 +1,298 @@ +/** + * Copyright 2000-2006 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.vocalizations; + +import java.io.IOException; +import java.util.Arrays; +import java.util.LinkedList; + +import javax.sound.sampled.AudioFileFormat; +import javax.sound.sampled.AudioFormat; +import javax.sound.sampled.AudioInputStream; + +import marytts.exceptions.MaryConfigurationException; +import marytts.exceptions.SynthesisException; +import marytts.signalproc.adaptation.prosody.BasicProsodyModifierParams; +import marytts.signalproc.analysis.RegularizedCepstrumEstimator; +import marytts.signalproc.sinusoidal.hntm.analysis.HntmAnalyzerParams; +import marytts.signalproc.sinusoidal.hntm.analysis.HntmSpeechSignal; +import marytts.signalproc.sinusoidal.hntm.synthesis.HntmSynthesizedSignal; +import marytts.signalproc.sinusoidal.hntm.synthesis.HntmSynthesizer; +import marytts.signalproc.sinusoidal.hntm.synthesis.HntmSynthesizerParams; +import marytts.unitselection.data.TimelineReader; +import marytts.util.data.BufferedDoubleDataSource; +import marytts.util.data.Datagram; +import marytts.util.data.DatagramDoubleDataSource; +import marytts.util.data.DoubleDataSource; +import marytts.util.data.audio.DDSAudioInputStream; +import marytts.util.math.MathUtils; +import marytts.util.math.Polynomial; + +/** + * HNM Synthesis technology to synthesize vocalizations + * + * @author Sathish Pammi + */ + +public class HNMSynthesisTechnology extends VocalizationSynthesisTechnology { + + protected HNMFeatureFileReader vHNMFeaturesReader; + protected VocalizationIntonationReader vIntonationReader; + protected HntmAnalyzerParams analysisParams; + protected HntmSynthesizerParams synthesisParams; + protected TimelineReader audioTimeline; + protected VocalizationUnitFileReader unitFileReader; + protected boolean f0ContourImposeSupport; + + public HNMSynthesisTechnology(String waveTimeLineFile, String unitFile, String hnmFeatureFile, String intonationFeatureFile, + boolean imposeF0Support) throws MaryConfigurationException { + + try { + this.audioTimeline = new TimelineReader(waveTimeLineFile); + this.unitFileReader = new VocalizationUnitFileReader(unitFile); + this.f0ContourImposeSupport = imposeF0Support; + this.vHNMFeaturesReader = new HNMFeatureFileReader(hnmFeatureFile); + + if (f0ContourImposeSupport) { + this.vIntonationReader = new VocalizationIntonationReader(intonationFeatureFile); + } else { + this.vIntonationReader = null; + } + } catch (IOException e) { + throw new MaryConfigurationException("Can not read data from files " + e); + } + + initializeParameters(); + } + + public HNMSynthesisTechnology(TimelineReader audioTimeline, VocalizationUnitFileReader unitFileReader, + HNMFeatureFileReader vHNMFeaturesReader, VocalizationIntonationReader vIntonationReader, boolean imposeF0Support) { + + this.audioTimeline = audioTimeline; + this.unitFileReader = unitFileReader; + this.vHNMFeaturesReader = vHNMFeaturesReader; + this.vIntonationReader = vIntonationReader; + this.f0ContourImposeSupport = imposeF0Support; + + initializeParameters(); + } + + /** + * intialize hnm parameters + */ + private void initializeParameters() { + // Analysis parameters + analysisParams = new HntmAnalyzerParams(); + analysisParams.harmonicModel = HntmAnalyzerParams.HARMONICS_PLUS_NOISE; + analysisParams.noiseModel = HntmAnalyzerParams.WAVEFORM; + analysisParams.useHarmonicAmplitudesDirectly = true; + analysisParams.harmonicSynthesisMethodBeforeNoiseAnalysis = HntmSynthesizerParams.LINEAR_PHASE_INTERPOLATION; + analysisParams.regularizedCepstrumWarpingMethod = RegularizedCepstrumEstimator.REGULARIZED_CEPSTRUM_WITH_POST_MEL_WARPING; + + // Synthesis parameters + synthesisParams = new HntmSynthesizerParams(); + synthesisParams.harmonicPartSynthesisMethod = HntmSynthesizerParams.LINEAR_PHASE_INTERPOLATION; + // synthesisParams.harmonicPartSynthesisMethod = HntmSynthesizerParams.QUADRATIC_PHASE_INTERPOLATION; + synthesisParams.overlappingHarmonicPartSynthesis = false; + synthesisParams.harmonicSynthesisOverlapInSeconds = 0.010f; + /* to output just one file */ + synthesisParams.writeHarmonicPartToSeparateFile = false; + synthesisParams.writeNoisePartToSeparateFile = false; + synthesisParams.writeTransientPartToSeparateFile = false; + synthesisParams.writeOriginalMinusHarmonicPartToSeparateFile = false; + } + + /** + * Synthesize given vocalization (i.e. unit-selection) + * + * @param unitIndex + * unit index + * @param aft + * audio file format + * @return AudioInputStream of synthesized vocalization + * @throws SynthesisException + * if failed to synthesize vocalization + */ + @Override + public AudioInputStream synthesize(int backchannelNumber, AudioFileFormat aft) throws SynthesisException { + + int numberOfBackChannels = unitFileReader.getNumberOfUnits(); + if (backchannelNumber >= numberOfBackChannels) { + throw new IllegalArgumentException("This voice has " + numberOfBackChannels + + " backchannels only. so it doesn't support unit number " + backchannelNumber); + } + + VocalizationUnit bUnit = unitFileReader.getUnit(backchannelNumber); + long start = bUnit.startTime; + int duration = bUnit.duration; + Datagram[] frames = null; + try { + frames = audioTimeline.getDatagrams(start, duration); + } catch (IOException e) { + throw new SynthesisException("Can not read data from timeline file " + e); + } + // Generate audio from frames + LinkedList datagrams = new LinkedList(); + datagrams.addAll(Arrays.asList(frames)); + DoubleDataSource audioSource = new DatagramDoubleDataSource(datagrams); + // audioSource.getAllData(); + return (new DDSAudioInputStream(new BufferedDoubleDataSource(audioSource), aft.getFormat())); + } + + /** + * Re-synthesize given vocalization using HNM technology + * + * @param unitIndex + * unit index + * @param aft + * audio file format + * @return AudioInputStream of synthesized vocalization + * @throws SynthesisException + * if failed to synthesize vocalization + */ + @Override + public AudioInputStream reSynthesize(int backchannelNumber, AudioFileFormat aft) throws SynthesisException { + float[] pScalesArray = { 1.0f }; + float[] tScalesArray = { 1.0f }; + float[] tScalesTimes = { 1.0f }; + float[] pScalesTimes = { 1.0f }; + return synthesizeUsingF0Modification(backchannelNumber, pScalesArray, pScalesTimes, tScalesArray, tScalesTimes, aft); + } + + /** + * Impose target intonation contour on given vocalization using HNM technology + * + * @param sourceIndex + * unit index of vocalization + * @param targetIndex + * unit index of target intonation + * @param aft + * audio file format + * @return AudioInputStream of synthesized vocalization + * @throws SynthesisException + * if failed to synthesize vocalization + */ + @Override + public AudioInputStream synthesizeUsingImposedF0(int sourceIndex, int targetIndex, AudioFileFormat aft) + throws SynthesisException { + + if (!f0ContourImposeSupport) { + throw new SynthesisException("Mary configuration of this voice doesn't support intonation contour imposition"); + } + + int numberOfUnits = vHNMFeaturesReader.getNumberOfUnits(); + if (sourceIndex >= numberOfUnits || targetIndex >= numberOfUnits) { + throw new IllegalArgumentException("sourceIndex(" + sourceIndex + ") and targetIndex(" + targetIndex + + ") are should be less than number of available units (" + numberOfUnits + ")"); + } + + double[] sourceF0 = this.vIntonationReader.getContour(sourceIndex); + double[] targetF0coeffs = this.vIntonationReader.getIntonationCoeffs(targetIndex); + double[] sourceF0coeffs = this.vIntonationReader.getIntonationCoeffs(sourceIndex); + + if (targetF0coeffs == null || sourceF0coeffs == null) { + return reSynthesize(sourceIndex, aft); + } + + if (targetF0coeffs.length == 0 || sourceF0coeffs.length == 0) { + return reSynthesize(sourceIndex, aft); + } + + double[] targetF0 = Polynomial.generatePolynomialValues(targetF0coeffs, sourceF0.length, 0, 1); + sourceF0 = Polynomial.generatePolynomialValues(sourceF0coeffs, sourceF0.length, 0, 1); + + assert targetF0.length == sourceF0.length; + float[] tScalesArray = { 1.0f }; + float[] tScalesTimes = { 1.0f }; + float[] pScalesArray = new float[targetF0.length]; + float[] pScalesTimes = new float[targetF0.length]; + double skipSizeInSeconds = this.vIntonationReader.getSkipSizeInSeconds(); + double windowSizeInSeconds = this.vIntonationReader.getWindowSizeInSeconds(); + for (int i = 0; i < targetF0.length; i++) { + pScalesArray[i] = (float) (targetF0[i] / sourceF0[i]); + pScalesTimes[i] = (float) (i * skipSizeInSeconds + 0.5 * windowSizeInSeconds); + } + + return synthesizeUsingF0Modification(sourceIndex, pScalesArray, pScalesTimes, tScalesArray, tScalesTimes, aft); + } + + /** + * modify intonation contour using HNM technology + * + * @param backchannelNumber + * unit index of vocalization + * @param pScalesArray + * pitch scales array + * @param pScalesTimes + * pitch scale times + * @param tScalesArray + * time scales array + * @param tScalesTimes + * time scale times + * @param aft + * audio file format + * @return AudioInputStream of synthesized vocalization + * @throws SynthesisException + * if failed to synthesize vocalization + */ + private AudioInputStream synthesizeUsingF0Modification(int backchannelNumber, float[] pScalesArray, float[] pScalesTimes, + float[] tScalesArray, float[] tScalesTimes, AudioFileFormat aft) throws SynthesisException { + + if (backchannelNumber > vHNMFeaturesReader.getNumberOfUnits()) { + throw new IllegalArgumentException("requesting unit should not be more than number of units"); + } + + if (!f0ContourImposeSupport) { + throw new SynthesisException("Mary configuration of this voice doesn't support intonation contour imposition"); + } + + BasicProsodyModifierParams pmodParams = new BasicProsodyModifierParams(tScalesArray, tScalesTimes, pScalesArray, + pScalesTimes); // Prosody from modification factors above + + HntmSpeechSignal hnmSignal = vHNMFeaturesReader.getHntmSpeechSignal(backchannelNumber); + HntmSynthesizer hs = new HntmSynthesizer(); + HntmSynthesizedSignal xhat = hs.synthesize(hnmSignal, null, null, pmodParams, null, analysisParams, synthesisParams); + + AudioFormat af; + if (aft == null) { // default audio format + float sampleRate = 16000.0F; // 8000,11025,16000,22050,44100 + int sampleSizeInBits = 16; // 8,16 + int channels = 1; // 1,2 + boolean signed = true; // true,false + boolean bigEndian = false; // true,false + af = new AudioFormat(sampleRate, sampleSizeInBits, channels, signed, bigEndian); + } else { + af = aft.getFormat(); + } + + double[] audio_double = xhat.output; + /* Normalise the signal before return, this will normalise between 1 and -1 */ + double MaxSample = MathUtils.getAbsMax(audio_double); + for (int i = 0; i < audio_double.length; i++) { + audio_double[i] = 0.3 * (audio_double[i] / MaxSample); + } + + // DDSAudioInputStream oais = new DDSAudioInputStream(new BufferedDoubleDataSource(audio_double), aft.getFormat()); + DDSAudioInputStream oais = new DDSAudioInputStream(new BufferedDoubleDataSource(audio_double), af); + + return oais; + } + +} diff --git a/marytts-purgatory/src/main/java/marytts/vocalizations/KMeansClusterer.java b/marytts-purgatory/src/main/java/marytts/vocalizations/KMeansClusterer.java new file mode 100644 index 0000000000..80785cce12 --- /dev/null +++ b/marytts-purgatory/src/main/java/marytts/vocalizations/KMeansClusterer.java @@ -0,0 +1,188 @@ +/** + * Copyright 2010 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.vocalizations; + +import java.awt.Color; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; + +import javax.sound.sampled.AudioFormat; +import javax.sound.sampled.AudioInputStream; +import javax.sound.sampled.AudioSystem; +import javax.sound.sampled.UnsupportedAudioFileException; +import javax.swing.JFrame; + +import marytts.machinelearning.KMeansClusteringTrainerParams; +import marytts.machinelearning.PolynomialCluster; +import marytts.machinelearning.PolynomialKMeansClusteringTrainer; +import marytts.signalproc.display.FunctionGraph; +import marytts.util.data.BufferedDoubleDataSource; +import marytts.util.data.audio.AudioDoubleDataSource; +import marytts.util.data.audio.AudioPlayer; +import marytts.util.data.audio.DDSAudioInputStream; +import marytts.util.math.Polynomial; + +public class KMeansClusterer { + + private ArrayList baseNames; + Polynomial[] f0Polynomials; + private int polynomialOrder = 3; + private int numberOfSamples = 0; + private int numberOfClusters; + private HashMap mapPolynomialBaseNames; + + public KMeansClusterer() { + baseNames = new ArrayList(); + mapPolynomialBaseNames = new HashMap(); + } + + public void loadF0Polynomials(String fileName) throws IOException { + + BufferedReader bfr = new BufferedReader(new FileReader(new File(fileName))); + String line; + ArrayList lines = new ArrayList(); + while ((line = bfr.readLine()) != null) { + lines.add(line.trim()); + } + double[][] f0PolynomialCoeffs; + String[] words = lines.get(0).trim().split("\\s+"); + polynomialOrder = words.length - 1; + numberOfSamples = lines.size(); + f0PolynomialCoeffs = new double[numberOfSamples][polynomialOrder]; + f0Polynomials = new Polynomial[numberOfSamples]; + + for (int i = 0; i < lines.size(); i++) { + line = lines.get(i); + words = line.trim().split("\\s+"); + baseNames.add(words[0].trim()); + + for (int j = 0; j < polynomialOrder; j++) { + f0PolynomialCoeffs[i][j] = (new Double(words[j + 1].trim())).doubleValue(); + } + + // System.out.println(f0PolynomialCoeffs[i][0]+" "+f0PolynomialCoeffs[i][1]+" "+f0PolynomialCoeffs[i][2]+" "+f0PolynomialCoeffs[i][3]); + // System.out.println(words[0]+" "+words[1]+" "+words[2]+" "+words[3]+" "+words[4]); + } + + // testing + for (int i = 0; i < numberOfSamples; i++) { + f0Polynomials[i] = new Polynomial(f0PolynomialCoeffs[i]); + mapPolynomialBaseNames.put(f0Polynomials[i], baseNames.get(i)); + System.out.print(baseNames.get(i) + " "); + for (int j = 0; j < polynomialOrder; j++) { + System.out.print(f0PolynomialCoeffs[i][j] + " "); + } + System.out.println(); + + double coeff[] = f0Polynomials[i].coeffs; + System.out.print(baseNames.get(i) + " "); + for (int j = 0; j < coeff.length; j++) { + System.out.print(coeff[j] + " "); + } + System.out.println(); + } + + } + + public void trainer(int numClusters) throws UnsupportedAudioFileException, IOException { + this.numberOfClusters = numClusters; + KMeansClusteringTrainerParams params = new KMeansClusteringTrainerParams(); + params.numClusters = numClusters; + params.maxIterations = 10000; + // Train: + PolynomialCluster[] clusters = PolynomialKMeansClusteringTrainer.train(f0Polynomials, params); + + // Visualise: + FunctionGraph clusterGraph = new FunctionGraph(0, 1, new double[1]); + clusterGraph.setYMinMax(-550, 500); + clusterGraph.setPrimaryDataSeriesStyle(Color.BLUE, FunctionGraph.DRAW_DOTS, FunctionGraph.DOT_FULLCIRCLE); + JFrame jf = clusterGraph.showInJFrame("", false, true); + for (int i = 0; i < clusters.length; i++) { + double[] meanValues = clusters[i].getMeanPolynomial().generatePolynomialValues(100, 0, 1); + clusterGraph.updateData(0, 1. / meanValues.length, meanValues); + + Polynomial[] members = clusters[i].getClusterMembers(); + System.out.print("Cluster " + i + " : "); + for (int m = 0; m < members.length; m++) { + double[] pred = members[m].generatePolynomialValues(meanValues.length, 0, 1); + clusterGraph.addDataSeries(pred, Color.GRAY, FunctionGraph.DRAW_LINE, -1); + String baseName = mapPolynomialBaseNames.get(members[m]); + System.out.print(baseName + " "); + jf.repaint(); + + String waveFile = "/home/sathish/Work/phd/voices/f0desc-listener/vocalizations/wav/" + File.separator + baseName + + ".wav"; + AudioInputStream inputAudio = AudioSystem.getAudioInputStream(new File(waveFile)); + + // Enforce PCM_SIGNED encoding + if (!inputAudio.getFormat().getEncoding().equals(AudioFormat.Encoding.PCM_SIGNED)) { + inputAudio = AudioSystem.getAudioInputStream(AudioFormat.Encoding.PCM_SIGNED, inputAudio); + } + + int audioSampleRate = (int) inputAudio.getFormat().getSampleRate(); + AudioDoubleDataSource signal = new AudioDoubleDataSource(inputAudio); + double[] sentenceAudio = signal.getAllData(); + AudioPlayer ap = new AudioPlayer(new DDSAudioInputStream(new BufferedDoubleDataSource(sentenceAudio), + new AudioFormat(AudioFormat.Encoding.PCM_SIGNED, audioSampleRate, // samples per second + 16, // bits per sample + 1, // mono + 2, // nr. of bytes per frame + audioSampleRate, // nr. of frames per second + true))); // big-endian;)) + ap.start(); + + try { + ap.join(); + Thread.sleep(10); + } catch (InterruptedException ie) { + } + + } + System.out.println(); + jf.setTitle("Cluster " + (i + 1) + " of " + clusters.length + ": " + members.length + " members"); + jf.repaint(); + + try { + Thread.sleep(5000); + } catch (InterruptedException ie) { + } + } + } + + /** + * @param args + * @throws IOException + * @throws UnsupportedAudioFileException + */ + public static void main(String[] args) throws IOException, UnsupportedAudioFileException { + KMeansClusterer kmc = new KMeansClusterer(); + // String fileName = "/home/sathish/phd/voices/en-GB-listener/vocal-polynomials/all.listener.polynomials.txt"; + // String fileName = "/home/sathish/phd/voices/en-GB-listener/vocal-polynomials/SpiVocalizationF0PolyFeatureFile.txt"; + String fileName = "/home/sathish/phd/voices/en-GB-listener/yeahPrudenceVocalizationF0PolyFeatureFile.txt"; + kmc.loadF0Polynomials(fileName); + kmc.trainer(5); + System.exit(0); + } + +} diff --git a/marytts-purgatory/src/main/java/marytts/vocalizations/MLSAFeatureFileReader.java b/marytts-purgatory/src/main/java/marytts/vocalizations/MLSAFeatureFileReader.java new file mode 100644 index 0000000000..7998516349 --- /dev/null +++ b/marytts-purgatory/src/main/java/marytts/vocalizations/MLSAFeatureFileReader.java @@ -0,0 +1,273 @@ +/** + * Portions Copyright 2006 DFKI GmbH. + * Portions Copyright 2001 Sun Microsystems, Inc. + * Portions Copyright 1999-2001 Language Technologies Institute, + * Carnegie Mellon University. + * All Rights Reserved. Use is subject to license terms. + * + * Permission is hereby granted, free of charge, to use and distribute + * this software and its documentation without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of this work, and to + * permit persons to whom this work is furnished to do so, subject to + * the following conditions: + * + * 1. The code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * 2. Any modifications must be clearly marked as such. + * 3. Original authors' names are not deleted. + * 4. The authors' names are not used to endorse or promote products + * derived from this software without specific prior written + * permission. + * + * DFKI GMBH AND THE CONTRIBUTORS TO THIS WORK DISCLAIM ALL WARRANTIES WITH + * REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL DFKI GMBH NOR THE + * CONTRIBUTORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL + * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR + * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS + * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF + * THIS SOFTWARE. + */ +package marytts.vocalizations; + +import java.io.BufferedInputStream; +import java.io.DataInputStream; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; + +import marytts.exceptions.MaryConfigurationException; +import marytts.util.data.MaryHeader; + +/** + * Reads a single file which contains all MLSA features (logfo, mgc and strengths) of vocalizations + * + * @author sathish pammi + * + */ +public class MLSAFeatureFileReader { + + private MaryHeader hdr = null; + + private int numberOfUnits; + private int STRVECTORSIZE; + private int MGCVECTORSIZE; + private int LF0VECTORSIZE; + + private int[] numberOfFrames; + private boolean[][] voiced; + private double[][] logf0; + private double[][][] strengths; + private double[][][] mgc; + + /** + * Create a feature file reader from the given MLSA feature file + * + * @param fileName + * the unit file to read + * @throws IOException + * if a problem occurs while reading + * @throws MaryConfigurationException + * if runtime configuration fails + */ + public MLSAFeatureFileReader(String fileName) throws IOException, MaryConfigurationException { + load(fileName); + } + + /** + * Load the given feature file + * + * @param fileName + * the feature file to read + * @throws IOException + * if a problem occurs while reading + * @throws MaryConfigurationException + * if runtime configuration fails + */ + private void load(String fileName) throws IOException, MaryConfigurationException { + // Open the file + DataInputStream dis = null; + + try { + dis = new DataInputStream(new BufferedInputStream(new FileInputStream(fileName))); + } catch (FileNotFoundException e) { + throw new MaryConfigurationException("File [" + fileName + "] was not found."); + } + + // Load the Mary header + hdr = new MaryHeader(dis); + if (hdr.getType() != MaryHeader.LISTENERFEATS) { + throw new MaryConfigurationException("File [" + fileName + "] is not a valid Mary Units file."); + } + + numberOfUnits = dis.readInt(); // Read the number of units + if (numberOfUnits < 0) { + throw new MaryConfigurationException("File [" + fileName + "] has a negative number of units. Aborting."); + } + + LF0VECTORSIZE = dis.readInt(); // Read LF0 vector size + MGCVECTORSIZE = dis.readInt(); // Read MGC vector size + STRVECTORSIZE = dis.readInt(); // Read STR vector size + + if (LF0VECTORSIZE != 1 || MGCVECTORSIZE <= 0 || STRVECTORSIZE <= 0) { + throw new MaryConfigurationException("File [" + fileName + + "] has no proper feature vector size information... Aborting."); + } + + logf0 = new double[numberOfUnits][]; + voiced = new boolean[numberOfUnits][]; + mgc = new double[numberOfUnits][][]; + strengths = new double[numberOfUnits][][]; + numberOfFrames = new int[numberOfUnits]; + + for (int i = 0; i < numberOfUnits; i++) { + + numberOfFrames[i] = dis.readInt(); + + // read LF0 data + int checkLF0Size = dis.readInt(); + assert checkLF0Size == (numberOfFrames[i] * LF0VECTORSIZE) : fileName + " feature file do not has proper format"; + logf0[i] = new double[numberOfFrames[i]]; + voiced[i] = new boolean[numberOfFrames[i]]; + for (int j = 0; j < numberOfFrames[i]; j++) { + logf0[i][j] = dis.readFloat(); + if (logf0[i][j] < 0) { + voiced[i][j] = false; + } else { + voiced[i][j] = true; + } + } + + // read MGC data + int checkMGCSize = dis.readInt(); + assert checkMGCSize == (numberOfFrames[i] * this.MGCVECTORSIZE) : fileName + " feature file do not has proper format"; + mgc[i] = new double[numberOfFrames[i]][MGCVECTORSIZE]; + for (int j = 0; j < numberOfFrames[i]; j++) { + for (int k = 0; k < MGCVECTORSIZE; k++) { + mgc[i][j][k] = dis.readFloat(); + } + } + + // read STR data + int checkSTRSize = dis.readInt(); + assert checkSTRSize == (numberOfFrames[i] * this.STRVECTORSIZE) : fileName + " feature file do not has proper format"; + strengths[i] = new double[numberOfFrames[i]][STRVECTORSIZE]; + for (int j = 0; j < numberOfFrames[i]; j++) { + for (int k = 0; k < STRVECTORSIZE; k++) { + strengths[i][j][k] = dis.readFloat(); + } + } + } + } + + /** + * Get the number of units in the file. + * + * @return The number of units. + */ + public int getNumberOfUnits() { + return (numberOfUnits); + } + + /** + * get boolean array of voiced frame information: true, if voiced; false if unvoiced; + * + * @param unitnumber + * unit index number + * @return boolean[] boolean array of voiced frames + * @throws IllegalArgumentException + * if given index number is not less than available units + */ + public boolean[] getVoicedFrames(int unitnumber) { + if (unitnumber >= this.numberOfUnits) { + throw new IllegalArgumentException("the given unit index number(" + unitnumber + + ") must be less than number of available units(" + this.numberOfUnits + ")"); + } + return this.voiced[unitnumber]; + } + + /** + * get array of logf0 features + * + * @param unitnumber + * unit index number + * @return double[] array of logf0 values + * @throws IllegalArgumentException + * if given index number is not less than available units + */ + public double[] getUnitLF0(int unitnumber) { + if (unitnumber >= this.numberOfUnits) { + throw new IllegalArgumentException("the given unit index number(" + unitnumber + + ") must be less than number of available units(" + this.numberOfUnits + ")"); + } + return this.logf0[unitnumber]; + } + + /** + * get double array of MGC features + * + * @param unitnumber + * unit index number + * @return double[][] array of mgc vectors + * @throws IllegalArgumentException + * if given index number is not less than available units + */ + public double[][] getUnitMGCs(int unitnumber) { + if (unitnumber >= this.numberOfUnits) { + throw new IllegalArgumentException("the given unit index number(" + unitnumber + + ") must be less than number of available units(" + this.numberOfUnits + ")"); + } + return this.mgc[unitnumber]; + } + + /** + * get double array of strength features + * + * @param unitnumber + * unit index number + * @return double[][] array of strength vectors + * @throws IllegalArgumentException + * if given index number is not less than available units + */ + public double[][] getUnitStrengths(int unitnumber) { + if (unitnumber >= this.numberOfUnits) { + throw new IllegalArgumentException("the given unit index number(" + unitnumber + + ") must be less than number of available units(" + this.numberOfUnits + ")"); + } + return this.strengths[unitnumber]; + } + + /** + * get vector size of MGC features + * + * @return int mgc vector size + */ + public int getMGCVectorSize() { + return this.MGCVECTORSIZE; + } + + /** + * get vector size of LF0 features + * + * @return int lf0 vector size + */ + public int getLF0VectorSize() { + return this.LF0VECTORSIZE; + } + + /** + * get vector size of strength features + * + * @return int strengths vector size + */ + public int getSTRVectorSize() { + return this.STRVECTORSIZE; + } + + public static void main(String[] args) throws Exception { + String fileName = "/home/sathish/Work/phd/voices/mlsa-poppy-listener/vocalizations/files/vocalization_mlsa_features.mry"; + MLSAFeatureFileReader bcUfr = new MLSAFeatureFileReader(fileName); + // bcUfr.load(fileName); + } +} diff --git a/marytts-purgatory/src/main/java/marytts/vocalizations/MLSASynthesisTechnology.java b/marytts-purgatory/src/main/java/marytts/vocalizations/MLSASynthesisTechnology.java new file mode 100644 index 0000000000..9064dbe7a4 --- /dev/null +++ b/marytts-purgatory/src/main/java/marytts/vocalizations/MLSASynthesisTechnology.java @@ -0,0 +1,283 @@ +/** + * Copyright 2000-2006 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.vocalizations; + +import java.io.FileInputStream; +import java.io.IOException; + +import javax.sound.sampled.AudioFileFormat; +import javax.sound.sampled.AudioFormat; +import javax.sound.sampled.AudioInputStream; + +import marytts.exceptions.MaryConfigurationException; +import marytts.exceptions.SynthesisException; +import marytts.htsengine.HMMData; +import marytts.htsengine.HTSPStream; +import marytts.htsengine.HTSVocoder; +import marytts.util.data.BufferedDoubleDataSource; +import marytts.util.data.audio.DDSAudioInputStream; +import marytts.util.math.MathUtils; +import marytts.util.math.Polynomial; + +/** + * MLSA Synthesis technology to synthesize vocalizations + * + * @author Sathish Pammi + */ + +public class MLSASynthesisTechnology extends VocalizationSynthesisTechnology { + + protected MLSAFeatureFileReader vMLSAFeaturesReader; + protected VocalizationIntonationReader vIntonationReader; + protected HMMData htsData; + protected HTSVocoder par2speech; + protected boolean imposePolynomialContour = true; + + public MLSASynthesisTechnology(String mlsaFeatureFile, String intonationFeatureFile, String mixedExcitationFile, + boolean imposePolynomialContour) throws MaryConfigurationException { + + try { + vMLSAFeaturesReader = new MLSAFeatureFileReader(mlsaFeatureFile); + if (intonationFeatureFile != null) { + this.vIntonationReader = new VocalizationIntonationReader(intonationFeatureFile); + } else { + this.vIntonationReader = null; + } + } catch (IOException ioe) { + throw new MaryConfigurationException("Problem with loading mlsa feature file", ioe); + } + + if (vMLSAFeaturesReader.getNumberOfUnits() <= 0) { + throw new MaryConfigurationException("mlsa feature file doesn't contain any data"); + } + + this.imposePolynomialContour = imposePolynomialContour; + + try { + htsData = new HMMData(); + htsData.setUseMixExc(true); + htsData.setUseFourierMag(false); /* use Fourier magnitudes for pulse generation */ + FileInputStream mixedFiltersStream = new FileInputStream(mixedExcitationFile); + htsData.setNumFilters(5); + htsData.setOrderFilters(48); + htsData.readMixedExcitationFilters(mixedFiltersStream); + htsData.setPdfStrStream(null); + // [min][max] + htsData.setF0Std(1.0); // variable for f0 control, multiply f0 [1.0][0.0--5.0] + htsData.setF0Mean(0.0); // variable for f0 control, add f0 [0.0][0.0--100.0] + } catch (Exception e) { + throw new MaryConfigurationException("htsData initialization failed.. ", e); + } + + par2speech = new HTSVocoder(); + + } + + /** + * Synthesize given vocalization using MLSA vocoder + * + * @param unitIndex + * unit index + * @param aft + * audio file format + * @return AudioInputStream of synthesized vocalization + * @throws SynthesisException + * if failed to synthesize vocalization + */ + public AudioInputStream synthesize(int backchannelNumber, AudioFileFormat aft) throws SynthesisException { + + if (backchannelNumber > vMLSAFeaturesReader.getNumberOfUnits()) { + throw new IllegalArgumentException("requesting unit should not be more than number of units"); + } + + if (backchannelNumber < 0) { + throw new IllegalArgumentException("requesting unit index should not be less than zero"); + } + + double[] lf0 = vMLSAFeaturesReader.getUnitLF0(backchannelNumber); + boolean[] voiced = vMLSAFeaturesReader.getVoicedFrames(backchannelNumber); + double[][] mgc = vMLSAFeaturesReader.getUnitMGCs(backchannelNumber); + double[][] strengths = vMLSAFeaturesReader.getUnitStrengths(backchannelNumber); + + return synthesizeUsingMLSAVocoder(mgc, strengths, lf0, voiced, aft); + } + + /** + * Re-synthesize given vocalization using MLSA (it is same as synthesize()) + * + * @param unitIndex + * unit index + * @param aft + * audio file format + * @return AudioInputStream of synthesized vocalization + * @throws SynthesisException + * if failed to synthesize vocalization + */ + @Override + public AudioInputStream reSynthesize(int backchannelNumber, AudioFileFormat aft) throws SynthesisException { + return synthesize(backchannelNumber, aft); + } + + /** + * Impose target intonation contour on given vocalization using MLSA technology + * + * @param sourceIndex + * unit index of vocalization + * @param targetIndex + * unit index of target intonation + * @param aft + * audio file format + * @return AudioInputStream of synthesized vocalization + * @throws SynthesisException + * if failed to synthesize vocalization + */ + @Override + public AudioInputStream synthesizeUsingImposedF0(int sourceIndex, int targetIndex, AudioFileFormat aft) + throws SynthesisException { + + if (sourceIndex > vMLSAFeaturesReader.getNumberOfUnits() || targetIndex > vMLSAFeaturesReader.getNumberOfUnits()) { + throw new IllegalArgumentException("requesting unit should not be more than number of units"); + } + + if (sourceIndex < 0 || targetIndex < 0) { + throw new IllegalArgumentException("requesting unit index should not be less than zero"); + } + + boolean[] voiced = vMLSAFeaturesReader.getVoicedFrames(sourceIndex); + double[][] mgc = vMLSAFeaturesReader.getUnitMGCs(sourceIndex); + double[][] strengths = vMLSAFeaturesReader.getUnitStrengths(sourceIndex); + + double[] lf0 = null; + + if (!this.imposePolynomialContour) { + lf0 = MathUtils.arrayResize(vMLSAFeaturesReader.getUnitLF0(targetIndex), voiced.length); + } else { + double[] targetF0coeffs = this.vIntonationReader.getIntonationCoeffs(targetIndex); + double[] sourceF0coeffs = this.vIntonationReader.getIntonationCoeffs(sourceIndex); + if (targetF0coeffs == null || sourceF0coeffs == null) { + return reSynthesize(sourceIndex, aft); + } + + if (targetF0coeffs.length == 0 || sourceF0coeffs.length == 0) { + return reSynthesize(sourceIndex, aft); + } + double[] f0Contour = Polynomial.generatePolynomialValues(targetF0coeffs, voiced.length, 0, 1); + lf0 = new double[f0Contour.length]; + for (int i = 0; i < f0Contour.length; i++) { + lf0[i] = Math.log(f0Contour[i]); + } + } + + return synthesizeUsingMLSAVocoder(mgc, strengths, lf0, voiced, aft); + } + + /** + * Synthesize using MLSA vocoder + * + * @param mgc + * mgc features + * @param strengths + * strengths + * @param lf0 + * logf0 features + * @param voiced + * voiced frames + * @param aft + * audio file format + * @return AudioInputStream of synthesized vocalization + * @throws SynthesisException + * if failed to synthesize vocalization + * @throws SynthesisException + * if log f0 values for voiced frames are not in the natural pitch range ( 30 to 1000 in Hertzs) + */ + private AudioInputStream synthesizeUsingMLSAVocoder(double[][] mgc, double[][] strengths, double[] lf0, boolean[] voiced, + AudioFileFormat aft) throws SynthesisException { + + assert lf0.length == mgc.length; + assert mgc.length == strengths.length; + + for (int i = 0; i < lf0.length; i++) { + if (lf0[i] > 0 && (Math.log(30) > lf0[i] || Math.log(1000) < lf0[i])) { + throw new SynthesisException("given log f0 values should be in the natural pitch range "); + } + } + + int mcepVsize = vMLSAFeaturesReader.getMGCVectorSize(); + int lf0Vsize = vMLSAFeaturesReader.getLF0VectorSize(); + int strVsize = vMLSAFeaturesReader.getSTRVectorSize(); + + HTSPStream lf0Pst = null; + HTSPStream mcepPst = null; + HTSPStream strPst = null; + + try { + lf0Pst = new HTSPStream(lf0Vsize * 3, lf0.length, HMMData.FeatureType.LF0, 0); // multiplied by 3 required for + // real-time synthesis + mcepPst = new HTSPStream(mcepVsize * 3, mgc.length, HMMData.FeatureType.MGC, 0); + strPst = new HTSPStream(strVsize * 3, strengths.length, HMMData.FeatureType.STR, 0); + } catch (Exception e) { + throw new SynthesisException("HTSPStream initialiaztion failed.. " + e); + } + + int lf0VoicedFrame = 0; + for (int i = 0; i < lf0.length; i++) { + if (voiced[i]) { + lf0Pst.setPar(lf0VoicedFrame, 0, lf0[i]); + lf0VoicedFrame++; + } + + for (int j = 0; j < mcepPst.getOrder(); j++) { + mcepPst.setPar(i, j, mgc[i][j]); + } + + for (int j = 0; j < strPst.getOrder(); j++) { + strPst.setPar(i, j, strengths[i][j]); + } + } + + AudioFormat af; + if (aft == null) { // default audio format + float sampleRate = 16000.0F; // 8000,11025,16000,22050,44100 + int sampleSizeInBits = 16; // 8,16 + int channels = 1; // 1,2 + boolean signed = true; // true,false + boolean bigEndian = false; // true,false + af = new AudioFormat(sampleRate, sampleSizeInBits, channels, signed, bigEndian); + } else { + af = aft.getFormat(); + } + + double[] audio_double = null; + try { + audio_double = par2speech.htsMLSAVocoder(lf0Pst, mcepPst, strPst, null, voiced, htsData, null); + } catch (Exception e) { + throw new SynthesisException("MLSA vocoding failed .. " + e); + } + + /* Normalise the signal before return, this will normalise between 1 and -1 */ + double MaxSample = MathUtils.getAbsMax(audio_double); + for (int i = 0; i < audio_double.length; i++) { + audio_double[i] = 0.3 * (audio_double[i] / MaxSample); + } + + return new DDSAudioInputStream(new BufferedDoubleDataSource(audio_double), af); + } + +} diff --git a/marytts-purgatory/src/main/java/marytts/vocalizations/SourceTargetPair.java b/marytts-purgatory/src/main/java/marytts/vocalizations/SourceTargetPair.java new file mode 100644 index 0000000000..a5e79d1d03 --- /dev/null +++ b/marytts-purgatory/src/main/java/marytts/vocalizations/SourceTargetPair.java @@ -0,0 +1,65 @@ +/** + * Copyright 2010 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.vocalizations; + +/** + * Class represents Source unit, target unit and contour distance between these units. Array of these pairs can sort with contour + * distance + * + * @author sathish + * + */ +public class SourceTargetPair implements Comparable { + + private int targetUnitIndex; + private int sourceUnitIndex; + private double distance; + + public SourceTargetPair(int sourceUnitIndex, int targetUnitIndex, double distance) { + this.sourceUnitIndex = sourceUnitIndex; + this.targetUnitIndex = targetUnitIndex; + this.distance = distance; + } + + public int compareTo(SourceTargetPair other) { + if (distance == other.distance) + return 0; + if (distance < other.distance) + return -1; + return 1; + } + + public boolean equals(Object dc) { + if (!(dc instanceof SourceTargetPair)) + return false; + SourceTargetPair other = (SourceTargetPair) dc; + if (distance == other.distance) + return true; + return false; + } + + public int getTargetUnitIndex() { + return this.targetUnitIndex; + } + + public int getSourceUnitIndex() { + return this.sourceUnitIndex; + } +} diff --git a/marytts-purgatory/src/main/java/marytts/vocalizations/VocalizationCandidate.java b/marytts-purgatory/src/main/java/marytts/vocalizations/VocalizationCandidate.java new file mode 100644 index 0000000000..523fc2cb3f --- /dev/null +++ b/marytts-purgatory/src/main/java/marytts/vocalizations/VocalizationCandidate.java @@ -0,0 +1,58 @@ +/** + * Copyright 2010 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.vocalizations; + +/** + * Class represents a vocalization candidate + * + * @author sathish + */ +public class VocalizationCandidate implements Comparable { + + int unitIndex; + double cost; + + public VocalizationCandidate(int unitIndex, double cost) { + this.unitIndex = unitIndex; + this.cost = cost; + } + + public int compareTo(VocalizationCandidate other) { + if (cost == other.cost) + return 0; + if (cost < other.cost) + return -1; + return 1; + } + + public boolean equals(Object dc) { + if (!(dc instanceof VocalizationCandidate)) + return false; + VocalizationCandidate other = (VocalizationCandidate) dc; + if (cost == other.cost) + return true; + return false; + } + + public String toString() { + return unitIndex + " " + cost; + } + +} diff --git a/marytts-purgatory/src/main/java/marytts/vocalizations/VocalizationFeatureFileReader.java b/marytts-purgatory/src/main/java/marytts/vocalizations/VocalizationFeatureFileReader.java new file mode 100644 index 0000000000..a66044cf0a --- /dev/null +++ b/marytts-purgatory/src/main/java/marytts/vocalizations/VocalizationFeatureFileReader.java @@ -0,0 +1,89 @@ +/** + * Copyright 2010 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.vocalizations; + +import java.io.BufferedInputStream; +import java.io.DataInputStream; +import java.io.FileInputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; + +import marytts.exceptions.MaryConfigurationException; +import marytts.features.FeatureDefinition; +import marytts.features.FeatureVector; +import marytts.util.data.MaryHeader; + +public class VocalizationFeatureFileReader extends marytts.unitselection.data.FeatureFileReader { + + public VocalizationFeatureFileReader(String fileName) throws IOException, MaryConfigurationException { + load(fileName); + } + + @Override + protected void loadFromStream(String fileName) throws IOException, MaryConfigurationException { + /* Open the file */ + DataInputStream dis = null; + dis = new DataInputStream(new BufferedInputStream(new FileInputStream(fileName))); + + /* Load the Mary header */ + hdr = new MaryHeader(dis); + if (hdr.getType() != MaryHeader.LISTENERFEATS) { + throw new MaryConfigurationException("File [" + fileName + "] is not a valid Mary listener feature file."); + } + featureDefinition = new FeatureDefinition(dis); + int numberOfUnits = dis.readInt(); + featureVectors = new FeatureVector[numberOfUnits]; + for (int i = 0; i < numberOfUnits; i++) { + featureVectors[i] = featureDefinition.readFeatureVector(i, dis); + } + } + + @Override + protected void loadFromByteBuffer(String fileName) throws IOException, MaryConfigurationException { + /* Open the file */ + /* Open the file */ + FileInputStream fis = new FileInputStream(fileName); + FileChannel fc = fis.getChannel(); + ByteBuffer bb = fc.map(FileChannel.MapMode.READ_ONLY, 0, fc.size()); + fis.close(); + + /* Load the Mary header */ + hdr = new MaryHeader(bb); + if (hdr.getType() != MaryHeader.LISTENERFEATS) { + throw new MaryConfigurationException("File [" + fileName + "] is not a valid Mary listener feature file."); + } + featureDefinition = new FeatureDefinition(bb); + int numberOfUnits = bb.getInt(); + featureVectors = new FeatureVector[numberOfUnits]; + for (int i = 0; i < numberOfUnits; i++) { + featureVectors[i] = featureDefinition.readFeatureVector(i, bb); + } + } + + /** + * @param args + */ + public static void main(String[] args) { + // TODO Auto-generated method stub + + } + +} diff --git a/marytts-purgatory/src/main/java/marytts/vocalizations/VocalizationIntonationReader.java b/marytts-purgatory/src/main/java/marytts/vocalizations/VocalizationIntonationReader.java new file mode 100644 index 0000000000..74a1c5690a --- /dev/null +++ b/marytts-purgatory/src/main/java/marytts/vocalizations/VocalizationIntonationReader.java @@ -0,0 +1,200 @@ +/** + * Portions Copyright 2006 DFKI GmbH. + * Portions Copyright 2001 Sun Microsystems, Inc. + * Portions Copyright 1999-2001 Language Technologies Institute, + * Carnegie Mellon University. + * All Rights Reserved. Use is subject to license terms. + * + * Permission is hereby granted, free of charge, to use and distribute + * this software and its documentation without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of this work, and to + * permit persons to whom this work is furnished to do so, subject to + * the following conditions: + * + * 1. The code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * 2. Any modifications must be clearly marked as such. + * 3. Original authors' names are not deleted. + * 4. The authors' names are not used to endorse or promote products + * derived from this software without specific prior written + * permission. + * + * DFKI GMBH AND THE CONTRIBUTORS TO THIS WORK DISCLAIM ALL WARRANTIES WITH + * REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL DFKI GMBH NOR THE + * CONTRIBUTORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL + * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR + * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS + * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF + * THIS SOFTWARE. + */ +package marytts.vocalizations; + +import java.io.BufferedInputStream; +import java.io.DataInputStream; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.Arrays; + +import marytts.exceptions.MaryConfigurationException; +import marytts.util.data.MaryHeader; + +/** + * Vocalization contours and their corresponding polynomial coeffs reader from a intonation timeline reader file + * + * @author sathish pammi + * + */ +public class VocalizationIntonationReader { + + private MaryHeader hdr = null; + private int numberOfUnits = 0; + private double windowSize = 0.0; + private double skipSize = 0.0; + private double[][] contours; + private double[][] coeffs; + private boolean[][] voiced; + + /****************/ + /* CONSTRUCTORS */ + /****************/ + + /** + * Create a unit file reader from the given unit file + * + * @param fileName + * the unit file to read + * @throws IOException + * if a problem occurs while reading + */ + public VocalizationIntonationReader(String fileName) throws IOException, MaryConfigurationException { + load(fileName); + } + + /** + * Load the given unit file + * + * @param fileName + * the unit file to read + * @throws IOException + * if a problem occurs while reading + */ + public void load(String fileName) throws IOException, MaryConfigurationException { + /* Open the file */ + DataInputStream dis = null; + + try { + dis = new DataInputStream(new BufferedInputStream(new FileInputStream(fileName))); + } catch (FileNotFoundException e) { + throw new RuntimeException("File [" + fileName + "] was not found."); + } + + try { + /* Load the Mary header */ + hdr = new MaryHeader(dis); + if (hdr.getType() != MaryHeader.LISTENERFEATS) { + throw new RuntimeException("File [" + fileName + "] is not a valid Mary Units file."); + } + + windowSize = dis.readFloat(); + + if (this.windowSize <= 0) { + throw new RuntimeException("File [" + fileName + "] has a negative number for window size. Aborting."); + } + + skipSize = dis.readFloat(); + if (this.skipSize <= 0) { + throw new RuntimeException("File [" + fileName + "] has a negative number for window size. Aborting."); + } + + /* Read the number of units */ + numberOfUnits = dis.readInt(); + if (numberOfUnits < 0) { + throw new RuntimeException("File [" + fileName + "] has a negative number of units. Aborting."); + } + + contours = new double[numberOfUnits][]; + coeffs = new double[numberOfUnits][]; + voiced = new boolean[numberOfUnits][]; + + for (int i = 0; i < numberOfUnits; i++) { + + // read polynomial coeffs + int polyOrder = dis.readInt(); + coeffs[i] = new double[polyOrder]; + for (int j = 0; j < polyOrder; j++) { + coeffs[i][j] = dis.readFloat(); + } + + // read f0 contour + int f0ContourLength = dis.readInt(); + contours[i] = new double[f0ContourLength]; + + voiced[i] = new boolean[f0ContourLength]; + Arrays.fill(voiced[i], false); + for (int j = 0; j < f0ContourLength; j++) { + contours[i][j] = dis.readFloat(); + if (contours[i][j] > 0) { + voiced[i][j] = true; + } + } + } + } catch (IOException e) { + throw new RuntimeException("Reading the Mary header from file [" + fileName + "] failed.", e); + } + + } + + /*****************/ + /* OTHER METHODS */ + /*****************/ + + /** + * Get the number of units in the file. + * + * @return The number of units. + */ + public int getNumberOfUnits() { + return (numberOfUnits); + } + + /** + * get an intonation contour + * + * @param unitIndexNumber + * @return + */ + public double[] getContour(int unitIndexNumber) { + return this.contours[unitIndexNumber]; + } + + public boolean[] getVoicings(int unitIndexNumber) { + return this.voiced[unitIndexNumber]; + } + + public double getWindowSizeInSeconds() { + return this.windowSize; + } + + public double getSkipSizeInSeconds() { + return this.skipSize; + } + + /** + * get an intonation polynomial coeffs + * + * @param unitIndexNumber + * @return + */ + public double[] getIntonationCoeffs(int unitIndexNumber) { + return this.coeffs[unitIndexNumber]; + } + + public static void main(String[] args) throws Exception { + String fileName = "/home/sathish/Work/phd/voices/listener/vocalizations/timelines/vocalization_intonation_timeline.mry"; + VocalizationIntonationReader bcUfr = new VocalizationIntonationReader(fileName); + // bcUfr.load(fileName); + } +} diff --git a/marytts-purgatory/src/main/java/marytts/vocalizations/VocalizationSelector.java b/marytts-purgatory/src/main/java/marytts/vocalizations/VocalizationSelector.java new file mode 100644 index 0000000000..b034358d9d --- /dev/null +++ b/marytts-purgatory/src/main/java/marytts/vocalizations/VocalizationSelector.java @@ -0,0 +1,639 @@ +/** + * Copyright 2010 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.vocalizations; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.text.DecimalFormat; +import java.util.Arrays; +import java.util.List; + +import marytts.exceptions.MaryConfigurationException; +import marytts.features.FeatureDefinition; +import marytts.features.FeatureVector; +import marytts.modules.synthesis.Voice; +import marytts.server.MaryProperties; +import marytts.unitselection.data.Unit; +import marytts.unitselection.select.Target; +import marytts.unitselection.select.VocalizationFFRTargetCostFunction; +import marytts.util.MaryUtils; +import marytts.util.math.MathUtils; +import marytts.util.math.Polynomial; + +import org.apache.log4j.Level; +import org.apache.log4j.Logger; +import org.w3c.dom.Element; + +/** + * Select suitable vocalization for a given target using a cost function + * + * @author sathish + * + */ +public class VocalizationSelector { + + protected VocalizationFeatureFileReader featureFileReader; + protected FeatureDefinition featureDefinition; + protected FeatureDefinition f0FeatureDefinition; + protected VocalizationIntonationReader vIntonationReader; + protected VocalizationUnitFileReader unitFileReader; + protected VocalizationFFRTargetCostFunction vffrtUnitCostFunction = null; + protected VocalizationFFRTargetCostFunction vffrtContourCostFunction = null; + protected boolean f0ContourImposeSupport; + protected boolean usePrecondition; + protected double contourCostWeight; + private DecimalFormat df; + + protected int noOfSuitableUnits = 1; + + protected Logger logger = MaryUtils.getLogger("Vocalization Selector"); + + public VocalizationSelector(Voice voice) throws MaryConfigurationException { + + String unitFileName = MaryProperties.getFilename("voice." + voice.getName() + ".vocalization.unitfile"); + String featureFile = MaryProperties.getFilename("voice." + voice.getName() + ".vocalization.featurefile"); + String featureDefinitionFile = MaryProperties.getFilename("voice." + voice.getName() + + ".vocalization.featureDefinitionFile"); + f0ContourImposeSupport = MaryProperties.getBoolean("voice." + voice.getName() + ".f0ContourImposeSupport", false); + df = new DecimalFormat("##.##"); + try { + BufferedReader fDBufferedReader = new BufferedReader(new FileReader(new File(featureDefinitionFile))); + this.featureDefinition = new FeatureDefinition(fDBufferedReader, true); + this.featureFileReader = new VocalizationFeatureFileReader(featureFile); + vffrtUnitCostFunction = new VocalizationFFRTargetCostFunction(this.featureFileReader, this.featureDefinition); + unitFileReader = new VocalizationUnitFileReader(unitFileName); + + if (this.featureFileReader.getNumberOfUnits() != this.unitFileReader.getNumberOfUnits()) { + throw new MaryConfigurationException("Feature file reader and unit file reader is not aligned properly"); + } + + if (this.f0ContourImposeSupport) { + String intonationFDFile = MaryProperties.getFilename("voice." + voice.getName() + + ".vocalization.intonation.featureDefinitionFile"); + String intonationFile = MaryProperties.getFilename("voice." + voice.getName() + ".vocalization.intonationfile"); + usePrecondition = MaryProperties.getBoolean("voice." + voice.getName() + ".vocalization.usePrecondition", false); + contourCostWeight = (new Double(MaryProperties.getProperty("voice." + voice.getName() + + ".vocalization.contourCostWeight", "0.5"))).doubleValue(); + if (contourCostWeight < 0 || contourCostWeight > 1.0) { + throw new MaryConfigurationException("contourCostWeight should be between 0 and 1"); + } + BufferedReader f0FDBufferedReader = new BufferedReader(new FileReader(new File(intonationFDFile))); + f0FeatureDefinition = new FeatureDefinition(f0FDBufferedReader, true); + vIntonationReader = new VocalizationIntonationReader(intonationFile); + noOfSuitableUnits = MaryProperties.getInteger("voice." + voice.getName() + + ".vocalization.intonation.numberOfSuitableUnits"); + vffrtContourCostFunction = new VocalizationFFRTargetCostFunction(this.featureFileReader, this.f0FeatureDefinition); + } + } catch (IOException e) { + throw new MaryConfigurationException("Problem loading vocalization files for voice ", e); + } + } + + /** + * Get feature definition used to select suitable candidate + * + * @return Feature Definition + */ + public FeatureDefinition getFeatureDefinition() { + return this.featureDefinition; + } + + /** + * Get best candidate pair to impose F0 contour on other + * + * @param domElement + * xml request for vocalization + * @return SourceTargetPair best candidate pair + */ + public SourceTargetPair getBestCandidatePairtoImposeF0(Element domElement) { + + VocalizationCandidate[] vCosts = getBestMatchingCandidates(domElement); + VocalizationCandidate[] vIntonationCosts = getBestIntonationCandidates(domElement); + + int noOfSuitableF0Units; + if (usePrecondition) { + noOfSuitableF0Units = getNumberContoursAboveThreshold(vCosts, vIntonationCosts); + } else { + noOfSuitableF0Units = noOfSuitableUnits; + } + + if (noOfSuitableF0Units == 0) { + return new SourceTargetPair(vCosts[0].unitIndex, vCosts[0].unitIndex, 0); + } + + VocalizationCandidate[] suitableCandidates = new VocalizationCandidate[noOfSuitableUnits]; + System.arraycopy(vCosts, 0, suitableCandidates, 0, noOfSuitableUnits); + VocalizationCandidate[] suitableF0Candidates = new VocalizationCandidate[noOfSuitableF0Units]; + System.arraycopy(vIntonationCosts, 0, suitableF0Candidates, 0, noOfSuitableF0Units); + + Target targetUnit = createTarget(domElement); + if (logger.getEffectiveLevel().equals(Level.DEBUG)) { + debugLogCandidates(targetUnit, suitableCandidates, suitableF0Candidates); + } + + SourceTargetPair[] sortedImposeF0Data = vocalizationF0DistanceComputer(suitableCandidates, suitableF0Candidates, + domElement); + + return sortedImposeF0Data[0]; + } + + /** + * compute a threshold according to precondition and get number of contours above computed threshold + * + * formula : for all CC(j) < CCmax where j1, j2, j3 ... are contour candidates CCmax (threshold) = min (CC(i1), CC(i2), + * CC(i3)....) where i1, i2, i3 .. are unit candidates + * + * @param vCosts + * VocalizationCandidates + * @param vIntonationCosts + * VocalizationF0Candidates + * @return int number of contours above computed threshold + */ + private int getNumberContoursAboveThreshold(VocalizationCandidate[] vCosts, VocalizationCandidate[] vIntonationCosts) { + + // get minimum cc cost for all units + double[] costs = new double[noOfSuitableUnits]; + for (int i = 0; i < costs.length; i++) { + int unitIndex = vCosts[i].unitIndex; + VocalizationCandidate contourCandidate = getCandidateByIndex(vIntonationCosts, unitIndex); + if (contourCandidate != null) { + costs[i] = contourCandidate.cost; + } else { + costs[i] = Double.MAX_VALUE; + } + } + + double threshold = MathUtils.min(costs); + int contourSetSize = 0; + for (int i = 0; i < vIntonationCosts.length; i++) { + if (vIntonationCosts[i].cost < threshold) { + contourSetSize = i + 1; + } else { + break; + } + } + + return contourSetSize; + } + + /** + * search candidate by unitindex + * + * @param vIntonationCosts + * VocalizationF0Candidates + * @param unitIndex + * unitindex + * @return VocalizationCandidate of given index + */ + private VocalizationCandidate getCandidateByIndex(VocalizationCandidate[] vIntonationCosts, int unitIndex) { + + for (int i = 0; i < vIntonationCosts.length; i++) { + if (vIntonationCosts[i].unitIndex == unitIndex) { + return vIntonationCosts[i]; + } + } + return null; + } + + /** + * polynomial distance computer between two units + * + * @param suitableCandidates + * vocalization candidates + * @param suitableF0Candidates + * intonation candidates + * @return an array of candidate pairs + */ + private SourceTargetPair[] vocalizationF0DistanceComputer(VocalizationCandidate[] suitableCandidates, + VocalizationCandidate[] suitableF0Candidates, Element domElement) { + + int noPossibleImpositions = suitableCandidates.length * suitableF0Candidates.length; + SourceTargetPair[] imposeF0Data = new SourceTargetPair[noPossibleImpositions]; + int count = 0; + + for (int i = 0; i < suitableCandidates.length; i++) { + for (int j = 0; j < suitableF0Candidates.length; j++) { + int sourceIndex = suitableCandidates[i].unitIndex; + int targetIndex = suitableF0Candidates[j].unitIndex; + + double contourCost = getContourCostDistance(sourceIndex, targetIndex); + double mergeCost = getMergeCost(sourceIndex, targetIndex, domElement); + double cost = (contourCost * contourCostWeight) + (mergeCost * (1 - contourCostWeight)); + logger.debug("Unit Index " + sourceIndex + " & Contour Index " + targetIndex + " :: Countour cost: " + + df.format(contourCost) + " + Merge Cost: " + df.format(mergeCost) + " --> TotalCost: " + + df.format(cost)); + imposeF0Data[count++] = new SourceTargetPair(sourceIndex, targetIndex, cost); + } + } + + Arrays.sort(imposeF0Data); + + return imposeF0Data; + } + + /** + * Compute mergeCost for unit and contour candidates Formula = segmentalformCost(u(i)) + intonationCost(c(i)) + + * voiceQualityCost(u(i)) + 0.5 * (meaningCost(u(i)) + meaningCost(c(i)) ) + * + * @param sourceIndex + * unit index + * @param targetIndex + * unit index + * @return double merge cost + */ + private double getMergeCost(int sourceIndex, int targetIndex, Element domElement) { + + Target targetUnit = createTarget(domElement); + Target targetContour = createIntonationTarget(domElement); + + Unit unitCandidate = this.unitFileReader.getUnit(sourceIndex); + Unit contourCandidate = this.unitFileReader.getUnit(targetIndex); + + // unit features + double segmentalCost = vffrtUnitCostFunction.featureCost(targetUnit, unitCandidate, "name"); + double voiceQualityCost = vffrtUnitCostFunction.featureCost(targetUnit, unitCandidate, "voicequality"); + double meaningUnitCost = 0; + String[] meaningFeatureNames = vffrtUnitCostFunction.getFeatureDefinition().getContinuousFeatureNameArray(); + for (int i = 0; i < meaningFeatureNames.length; i++) { + meaningUnitCost += vffrtUnitCostFunction.featureCost(targetUnit, unitCandidate, meaningFeatureNames[i]); + } + + // contour features + double intonationCost = this.vffrtContourCostFunction.featureCost(targetContour, contourCandidate, "intonation"); + double meaningContourCost = 0; + String[] meaningContourFeatures = vffrtContourCostFunction.getFeatureDefinition().getContinuousFeatureNameArray(); + for (int i = 0; i < meaningContourFeatures.length; i++) { + meaningContourCost += vffrtContourCostFunction + .featureCost(targetContour, contourCandidate, meaningContourFeatures[i]); + } + + double mergeCost = segmentalCost + voiceQualityCost + intonationCost + 0.5 * (meaningUnitCost + meaningContourCost); + + return mergeCost; + } + + /** + * compute contour distance (polynomial distance) + * + * @param sourceIndex + * unit index + * @param targetIndex + * unit index + * @return polynomial distance measure + */ + private double getContourCostDistance(int sourceIndex, int targetIndex) { + double distance; + if (targetIndex == sourceIndex) { + distance = 0; + } else { + double[] targetCoeffs = vIntonationReader.getIntonationCoeffs(targetIndex); + double[] sourceCoeffs = vIntonationReader.getIntonationCoeffs(sourceIndex); + if (targetCoeffs != null && sourceCoeffs != null && targetCoeffs.length == sourceCoeffs.length) { + distance = Polynomial.polynomialDistance(sourceCoeffs, targetCoeffs); + } else { + distance = Double.MAX_VALUE; + } + } + return distance; + } + + /** + * get a best matching candidate for a given target + * + * @param domElement + * xml request for vocalization + * @return unit index of best matching candidate + */ + public int getBestMatchingCandidate(Element domElement) { + + Target targetUnit = createTarget(domElement); + int numberUnits = this.unitFileReader.getNumberOfUnits(); + double minCost = Double.MAX_VALUE; + int index = 0; + for (int i = 0; i < numberUnits; i++) { + Unit singleUnit = this.unitFileReader.getUnit(i); + double cost = vffrtUnitCostFunction.cost(targetUnit, singleUnit); + if (cost < minCost) { + minCost = cost; + index = i; + } + } + + return index; + } + + /** + * get a array of best candidates sorted according to cost + * + * @param domElement + * xml request for vocalization + * @return an array of best vocalization candidates + */ + public VocalizationCandidate[] getBestMatchingCandidates(Element domElement) { + // FeatureDefinition featDef = this.featureFileReader.getFeatureDefinition(); + Target targetUnit = createTarget(domElement); + int numberUnits = this.unitFileReader.getNumberOfUnits(); + VocalizationCandidate[] vocalizationCandidates = new VocalizationCandidate[numberUnits]; + for (int i = 0; i < numberUnits; i++) { + Unit singleUnit = this.unitFileReader.getUnit(i); + double cost = vffrtUnitCostFunction.cost(targetUnit, singleUnit); + vocalizationCandidates[i] = new VocalizationCandidate(i, cost); + } + Arrays.sort(vocalizationCandidates); + return vocalizationCandidates; + } + + /** + * Debug messages for selected candidates + * + * @param targetUnit + * target unit + * @param suitableCandidates + * suitable vocalization candidates + * @param suitableF0Candidates + * suitable intonation candidates + */ + private void debugLogCandidates(Target targetUnit, VocalizationCandidate[] suitableCandidates, + VocalizationCandidate[] suitableF0Candidates) { + FeatureVector targetFeatures = targetUnit.getFeatureVector(); + FeatureDefinition fd = featureFileReader.getFeatureDefinition(); + int fiName = fd.getFeatureIndex("name"); + int fiIntonation = fd.getFeatureIndex("intonation"); + int fiVQ = fd.getFeatureIndex("voicequality"); + for (int i = 0; i < suitableCandidates.length; i++) { + int unitIndex = suitableCandidates[i].unitIndex; + double unitCost = suitableCandidates[i].cost; + FeatureVector fv = featureFileReader.getFeatureVector(unitIndex); + StringBuilder sb = new StringBuilder(); + sb.append("Candidate ").append(i).append(": ").append(unitIndex).append(" ( " + unitCost + " ) ").append(" -- "); + byte bName = fv.getByteFeature(fiName); + if (fv.getByteFeature(fiName) != 0 && targetFeatures.getByteFeature(fiName) != 0) { + sb.append(" ").append(fv.getFeatureAsString(fiName, fd)); + } + if (fv.getByteFeature(fiVQ) != 0 && targetFeatures.getByteFeature(fiName) != 0) { + sb.append(" ").append(fv.getFeatureAsString(fiVQ, fd)); + } + if (fv.getByteFeature(fiIntonation) != 0 && targetFeatures.getByteFeature(fiIntonation) != 0) { + sb.append(" ").append(fv.getFeatureAsString(fiIntonation, fd)); + } + for (int j = 0; j < targetFeatures.getLength(); j++) { + if (targetFeatures.isContinuousFeature(j) && !Float.isNaN((Float) targetFeatures.getFeature(j)) + && !Float.isNaN((Float) fv.getFeature(j))) { + String featureName = fd.getFeatureName(j); + sb.append(" ").append(featureName).append("=").append(fv.getFeature(j)); + } + } + logger.debug(sb.toString()); + } + for (int i = 0; i < suitableF0Candidates.length; i++) { + int unitIndex = suitableF0Candidates[i].unitIndex; + double unitCost = suitableF0Candidates[i].cost; + FeatureVector fv = featureFileReader.getFeatureVector(unitIndex); + StringBuilder sb = new StringBuilder(); + sb.append("F0 Candidate ").append(i).append(": ").append(unitIndex).append(" ( " + unitCost + " ) ").append(" -- "); + byte bName = fv.getByteFeature(fiName); + if (fv.getByteFeature(fiName) != 0 && targetFeatures.getByteFeature(fiName) != 0) { + sb.append(" ").append(fv.getFeatureAsString(fiName, fd)); + } + if (fv.getByteFeature(fiVQ) != 0 && targetFeatures.getByteFeature(fiName) != 0) { + sb.append(" ").append(fv.getFeatureAsString(fiVQ, fd)); + } + if (fv.getByteFeature(fiIntonation) != 0 && targetFeatures.getByteFeature(fiIntonation) != 0) { + sb.append(" ").append(fv.getFeatureAsString(fiIntonation, fd)); + } + for (int j = 0; j < targetFeatures.getLength(); j++) { + if (targetFeatures.isContinuousFeature(j) && !Float.isNaN((Float) targetFeatures.getFeature(j)) + && !Float.isNaN((Float) fv.getFeature(j))) { + String featureName = fd.getFeatureName(j); + sb.append(" ").append(featureName).append("=").append(fv.getFeature(j)); + } + } + logger.debug(sb.toString()); + } + } + + /** + * get a array of best candidates sorted according to cost (cost computed on f0_feature_definition features only) + * + * @param domElement + * xml request for vocalization + * @return VocalizationCandidate[] a array of best candidates + */ + private VocalizationCandidate[] getBestIntonationCandidates(Element domElement) { + + Target targetUnit = createIntonationTarget(domElement); + int numberUnits = this.unitFileReader.getNumberOfUnits(); + VocalizationCandidate[] vocalizationCandidates = new VocalizationCandidate[numberUnits]; + for (int i = 0; i < numberUnits; i++) { + Unit singleUnit = this.unitFileReader.getUnit(i); + double cost = vffrtContourCostFunction.cost(targetUnit, singleUnit); + vocalizationCandidates[i] = new VocalizationCandidate(i, cost); + } + Arrays.sort(vocalizationCandidates); + return vocalizationCandidates; + } + + /** + * create target from XML request + * + * @param domElement + * xml request for vocalization + * @return Target target represents xml request + */ + private Target createTarget(Element domElement) { + + // FeatureDefinition featDef = this.featureFileReader.getFeatureDefinition(); + FeatureDefinition featDef = this.featureDefinition; + int numFeatures = featDef.getNumberOfFeatures(); + int numByteFeatures = featDef.getNumberOfByteFeatures(); + int numShortFeatures = featDef.getNumberOfShortFeatures(); + int numContiniousFeatures = featDef.getNumberOfContinuousFeatures(); + byte[] byteFeatures = new byte[numByteFeatures]; + short[] shortFeatures = new short[numShortFeatures]; + float[] floatFeatures = new float[numContiniousFeatures]; + int byteCount = 0; + int shortCount = 0; + int floatCount = 0; + + for (int i = 0; i < numFeatures; i++) { + + String featName = featDef.getFeatureName(i); + String featValue = "0"; + + if (featDef.isByteFeature(featName) || featDef.isShortFeature(featName)) { + if (domElement.hasAttribute(featName)) { + featValue = domElement.getAttribute(featName); + } + + boolean hasFeature = featDef.hasFeatureValue(featName, featValue); + if (!hasFeature) + featValue = "0"; + + if (featDef.isByteFeature(i)) { + byteFeatures[byteCount++] = featDef.getFeatureValueAsByte(i, featValue); + } else if (featDef.isShortFeature(i)) { + shortFeatures[shortCount++] = featDef.getFeatureValueAsShort(i, featValue); + } + } else { + if (domElement.hasAttribute("meaning")) { + featValue = domElement.getAttribute("meaning"); + } + // float contFeature = getMeaningScaleValue ( featName, featValue ); + floatFeatures[floatCount++] = getMeaningScaleValue(featName, featValue); + } + } + + FeatureVector newFV = featDef.toFeatureVector(0, byteFeatures, shortFeatures, floatFeatures); + + String name = "0"; + if (domElement.hasAttribute("name")) { + name = domElement.getAttribute("name"); + } + + Target newTarget = new Target(name, domElement); + newTarget.setFeatureVector(newFV); + + return newTarget; + } + + /** + * create F0 target from XML request + * + * @param domElement + * xml request for intonation + * @return Target target represents xml request + */ + private Target createIntonationTarget(Element domElement) { + + // FeatureDefinition featDef = this.featureFileReader.getFeatureDefinition(); + FeatureDefinition featDef = this.f0FeatureDefinition; + int numFeatures = featDef.getNumberOfFeatures(); + int numByteFeatures = featDef.getNumberOfByteFeatures(); + int numShortFeatures = featDef.getNumberOfShortFeatures(); + int numContiniousFeatures = featDef.getNumberOfContinuousFeatures(); + byte[] byteFeatures = new byte[numByteFeatures]; + short[] shortFeatures = new short[numShortFeatures]; + float[] floatFeatures = new float[numContiniousFeatures]; + int byteCount = 0; + int shortCount = 0; + int floatCount = 0; + + for (int i = 0; i < numFeatures; i++) { + + String featName = featDef.getFeatureName(i); + String featValue = "0"; + + if (featDef.isByteFeature(featName) || featDef.isShortFeature(featName)) { + if (domElement.hasAttribute(featName)) { + featValue = domElement.getAttribute(featName); + } + + boolean hasFeature = featDef.hasFeatureValue(featName, featValue); + if (!hasFeature) + featValue = "0"; + + if (featDef.isByteFeature(i)) { + byteFeatures[byteCount++] = featDef.getFeatureValueAsByte(i, featValue); + } else if (featDef.isShortFeature(i)) { + shortFeatures[shortCount++] = featDef.getFeatureValueAsShort(i, featValue); + } + } else { + if (domElement.hasAttribute("meaning")) { + featValue = domElement.getAttribute("meaning"); + } + // float contFeature = getMeaningScaleValue ( featName, featValue ); + floatFeatures[floatCount++] = getMeaningScaleValue(featName, featValue); + } + } + + FeatureVector newFV = featDef.toFeatureVector(0, byteFeatures, shortFeatures, floatFeatures); + + String name = "0"; + if (domElement.hasAttribute("name")) { + name = domElement.getAttribute("name"); + } + + Target newTarget = new Target(name, domElement); + newTarget.setFeatureVector(newFV); + + return newTarget; + } + + /** + * get value on meaning scale as a float value + * + * @param featureName + * feature names + * @param meaningAttribute + * meaning attribute + * @return a float value for a meaning feature + */ + private float getMeaningScaleValue(String featureName, String meaningAttribute) { + + String[] categories = meaningAttribute.split("\\s+"); + List categoriesList = Arrays.asList(categories); + + if ("anger".equals(featureName) && categoriesList.contains("anger")) { + return 5; + } else if ("sadness".equals(featureName) && categoriesList.contains("sadness")) { + return 5; + } else if ("amusement".equals(featureName) && categoriesList.contains("amusement")) { + return 5; + } else if ("happiness".equals(featureName) && categoriesList.contains("happiness")) { + return 5; + } else if ("contempt".equals(featureName) && categoriesList.contains("contempt")) { + return 5; + } else if ("certain".equals(featureName) && categoriesList.contains("uncertain")) { + return -2; + } else if ("certain".equals(featureName) && categoriesList.contains("certain")) { + return 2; + } else if ("agreeing".equals(featureName) && categoriesList.contains("disagreeing")) { + return -2; + } else if ("agreeing".equals(featureName) && categoriesList.contains("agreeing")) { + return 2; + } else if ("interested".equals(featureName) && categoriesList.contains("uninterested")) { + return -2; + } else if ("interested".equals(featureName) && categoriesList.contains("interested")) { + return 2; + } else if ("anticipation".equals(featureName) && categoriesList.contains("low-anticipation")) { + return -2; + } else if ("anticipation".equals(featureName) && categoriesList.contains("anticipation")) { + return 2; + } else if ("anticipation".equals(featureName) && categoriesList.contains("high-anticipation")) { + return 2; + } else if ("solidarity".equals(featureName) && categoriesList.contains("solidarity")) { + return 5; + } else if ("solidarity".equals(featureName) && categoriesList.contains("low-solidarity")) { + return 1; + } else if ("solidarity".equals(featureName) && categoriesList.contains("high-solidarity")) { + return 5; + } else if ("antagonism".equals(featureName) && categoriesList.contains("antagonism")) { + return 5; + } else if ("antagonism".equals(featureName) && categoriesList.contains("high-antagonism")) { + return 5; + } else if ("antagonism".equals(featureName) && categoriesList.contains("low-antagonism")) { + return 1; + } + + return Float.NaN; + } +} diff --git a/marytts-purgatory/src/main/java/marytts/vocalizations/VocalizationSynthesisTechnology.java b/marytts-purgatory/src/main/java/marytts/vocalizations/VocalizationSynthesisTechnology.java new file mode 100644 index 0000000000..3b4afbc72d --- /dev/null +++ b/marytts-purgatory/src/main/java/marytts/vocalizations/VocalizationSynthesisTechnology.java @@ -0,0 +1,77 @@ +/** + * Copyright 2000-2006 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.vocalizations; + +import javax.sound.sampled.AudioFileFormat; +import javax.sound.sampled.AudioInputStream; + +import marytts.exceptions.SynthesisException; + +/** + * An abstract class for vocalization syntehsis technology + * + * @author Sathish Pammi + */ + +public abstract class VocalizationSynthesisTechnology { + + /** + * Synthesize given vocalization + * + * @param unitIndex + * unit index + * @param aft + * audio file format + * @return AudioInputStream of synthesized vocalization + * @throws SynthesisException + * if failed to synthesize vocalization + */ + public abstract AudioInputStream synthesize(int unitIndex, AudioFileFormat aft) throws SynthesisException; + + /** + * Re-synthesize given vocalization + * + * @param unitIndex + * unit index + * @param aft + * audio file format + * @return AudioInputStream of synthesized vocalization + * @throws SynthesisException + * if failed to synthesize vocalization + */ + public abstract AudioInputStream reSynthesize(int sourceIndex, AudioFileFormat aft) throws SynthesisException; + + /** + * Impose target intonation contour on given vocalization + * + * @param sourceIndex + * unit index of vocalization + * @param targetIndex + * unit index of target intonation + * @param aft + * aft audio file format + * @return AudioInputStream of synthesized vocalization + * @throws SynthesisException + * if failed to synthesize vocalization + */ + public abstract AudioInputStream synthesizeUsingImposedF0(int sourceIndex, int targetIndex, AudioFileFormat aft) + throws SynthesisException; + +} diff --git a/marytts-purgatory/src/main/java/marytts/vocalizations/VocalizationSynthesizer.java b/marytts-purgatory/src/main/java/marytts/vocalizations/VocalizationSynthesizer.java new file mode 100644 index 0000000000..a358561816 --- /dev/null +++ b/marytts-purgatory/src/main/java/marytts/vocalizations/VocalizationSynthesizer.java @@ -0,0 +1,305 @@ +/** + * Copyright 2000-2006 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.vocalizations; + +import java.io.IOException; + +import javax.sound.sampled.AudioFileFormat; +import javax.sound.sampled.AudioInputStream; + +import marytts.datatypes.MaryXML; +import marytts.exceptions.MaryConfigurationException; +import marytts.exceptions.SynthesisException; +import marytts.features.FeatureDefinition; +import marytts.modules.synthesis.Voice; +import marytts.server.MaryProperties; +import marytts.unitselection.data.Unit; +import marytts.util.MaryUtils; + +import org.apache.log4j.Logger; +import org.w3c.dom.Element; + +/** + * The vocalization synthesis module. + * + * @author Sathish Pammi + */ + +public class VocalizationSynthesizer { + + protected VocalizationSynthesisTechnology vSynthesizer; + protected VocalizationSelector vSelector; + protected VocalizationUnitFileReader unitFileReader; + protected boolean f0ContourImposeSupport; + + protected Logger logger = MaryUtils.getLogger("Vocalization Synthesizer"); + + public VocalizationSynthesizer(Voice voice) throws MaryConfigurationException { + + if (!voice.hasVocalizationSupport()) { + throw new MaryConfigurationException("This voice " + voice.toString() + " doesn't support synthesis of vocalizations"); + } + + String unitFileName = MaryProperties.getFilename("voice." + voice.getName() + ".vocalization.unitfile"); + + try { + this.unitFileReader = new VocalizationUnitFileReader(unitFileName); + } catch (IOException e) { + throw new MaryConfigurationException("can't read unit file"); + } + + String intonationFile = MaryProperties.getFilename("voice." + voice.getName() + ".vocalization.intonationfile"); + String technology = MaryProperties.getProperty("voice." + voice.getName() + ".vocalization.synthesisTechnology", + "fdpsola"); + f0ContourImposeSupport = MaryProperties.getBoolean("voice." + voice.getName() + ".f0ContourImposeSupport", false); + + if ("fdpsola".equals(technology)) { + String timelineFile = MaryProperties.getFilename("voice." + voice.getName() + ".vocalization.timeline"); + vSynthesizer = new FDPSOLASynthesisTechnology(timelineFile, unitFileName, intonationFile, f0ContourImposeSupport); + } else if ("mlsa".equals(technology)) { + boolean imposePolynomialContour = MaryProperties.getBoolean("voice." + voice.getName() + + ".vocalization.imposePolynomialContour", true); + String mlsaFeatureFile = MaryProperties.getFilename("voice." + voice.getName() + ".vocalization.mlsafeaturefile"); + String mixedExcitationFilter = MaryProperties.getFilename("voice." + voice.getName() + + ".vocalization.mixedexcitationfilter"); + vSynthesizer = new MLSASynthesisTechnology(mlsaFeatureFile, intonationFile, mixedExcitationFilter, + imposePolynomialContour); + } else if ("hnm".equals(technology)) { + String timelineFile = MaryProperties.getFilename("voice." + voice.getName() + ".vocalization.timeline"); + String hnmFeatureFile = MaryProperties.getFilename("voice." + voice.getName() + ".vocalization.hnmfeaturefile"); + vSynthesizer = new HNMSynthesisTechnology(timelineFile, unitFileName, hnmFeatureFile, intonationFile, + f0ContourImposeSupport); + } else { + throw new MaryConfigurationException("the property 'voice." + voice.getName() + + ".vocalization.synthesisTechnology' should be one among 'hnm', 'mlsa' and 'fdpsola'"); + } + + this.vSelector = new VocalizationSelector(voice); + } + + /** + * Handle a request for synthesis of vocalization + * + * @param voice + * the selected voice + * @param aft + * AudioFileFormat of the output AudioInputStream + * @param domElement + * target xml element ('vocalization' element) + * @return AudioInputStream of requested vocalization it returns null if the voice doesn't support synthesis of vocalizations + * @throws IllegalArgumentException + * if domElement contains 'variant' attribute value is greater than available number of vocalizations + */ + public AudioInputStream synthesize(Voice voice, AudioFileFormat aft, Element domElement) throws Exception { + + if (!voice.hasVocalizationSupport()) + return null; + + if (domElement.hasAttribute("variant")) { + return synthesizeVariant(aft, domElement); + } + + if (f0ContourImposeSupport) { + return synthesizeImposedIntonation(aft, domElement); + } + + return synthesizeVocalization(aft, domElement); + } + + /** + * Synthesize a "variant" vocalization + * + * @param aft + * AudioFileFormat of the output AudioInputStream + * @param domElement + * target 'vocalization' xml element + * @return AudioInputStream of requested vocalization + * @throws SynthesisException + * if it can't synthesize vocalization + * @throws IllegalArgumentException + * if domElement contains 'variant' attribute value is greater than available number of vocalizations + */ + private AudioInputStream synthesizeVariant(AudioFileFormat aft, Element domElement) throws SynthesisException { + + int numberOfBackChannels = unitFileReader.getNumberOfUnits(); + int backchannelNumber = 0; + + if (domElement.hasAttribute("variant")) { + backchannelNumber = Integer.parseInt(domElement.getAttribute("variant")); + } + + if (backchannelNumber >= numberOfBackChannels) { + throw new IllegalArgumentException("This voice has " + numberOfBackChannels + + " backchannels only. so it doesn't support unit number " + backchannelNumber); + } + + return synthesizeSelectedVocalization(backchannelNumber, aft, domElement); + } + + /** + * Synthesize a vocalization which fits better for given target + * + * @param aft + * AudioFileFormat of the output AudioInputStream + * @param domElement + * target 'vocalization' xml element + * @return AudioInputStream output audio + * @throws SynthesisException + * if it can't synthesize vocalization + */ + private AudioInputStream synthesizeVocalization(AudioFileFormat aft, Element domElement) throws SynthesisException { + + int numberOfBackChannels = unitFileReader.getNumberOfUnits(); + int backchannelNumber = vSelector.getBestMatchingCandidate(domElement); + // here it is a bug, if getBestMatchingCandidate select a backchannelNumber greater than numberOfBackChannels + assert backchannelNumber < numberOfBackChannels : "This voice has " + numberOfBackChannels + + " backchannels only. so it doesn't support unit number " + backchannelNumber; + + return synthesizeSelectedVocalization(backchannelNumber, aft, domElement); + } + + /** + * Synthesize a vocalization which fits better for given target, in addition, impose intonation from closest best vocalization + * according to given feature definition for intonation selection + * + * @param aft + * AudioFileFormat of the output AudioInputStream + * @param domElement + * target 'vocalization' xml element + * @return AudioInputStream output audio + * @throws SynthesisException + * if it can't synthesize vocalization + */ + private AudioInputStream synthesizeImposedIntonation(AudioFileFormat aft, Element domElement) throws SynthesisException { + + SourceTargetPair imposeF0Data = vSelector.getBestCandidatePairtoImposeF0(domElement); + int targetIndex = imposeF0Data.getTargetUnitIndex(); + int sourceIndex = imposeF0Data.getSourceUnitIndex(); + + logger.debug("Synthesizing candidate " + sourceIndex + " with intonation contour " + targetIndex); + + if (targetIndex == sourceIndex) { + return synthesizeSelectedVocalization(sourceIndex, aft, domElement); + } + + return imposeF0ContourOnVocalization(sourceIndex, targetIndex, aft, domElement); + } + + /** + * Impose a target f0 contour onto a (source) unit + * + * @param sourceIndex + * unit index of segmentalform unit + * @param targetIndex + * unit index of target f0 contour + * @param aft + * AudioFileFormat of the output AudioInputStream + * @param domElement + * target 'vocalization' xml element + * @return AudioInputStream of requested vocalization + * @throws SynthesisException + * if no data can be read at the given target time or if audio processing fails + */ + private AudioInputStream imposeF0ContourOnVocalization(int sourceIndex, int targetIndex, AudioFileFormat aft, + Element domElement) throws SynthesisException { + + int numberOfBackChannels = unitFileReader.getNumberOfUnits(); + + if (targetIndex >= numberOfBackChannels) { + throw new IllegalArgumentException("This voice has " + numberOfBackChannels + + " backchannels only. so it doesn't support unit number " + targetIndex); + } + + if (sourceIndex >= numberOfBackChannels) { + throw new IllegalArgumentException("This voice has " + numberOfBackChannels + + " backchannels only. so it doesn't support unit number " + sourceIndex); + } + + VocalizationUnit bUnit = unitFileReader.getUnit(sourceIndex); + Unit[] units = bUnit.getUnits(); + String[] unitNames = bUnit.getUnitNames(); + long endTime = 0l; + for (int i = 0; i < units.length; i++) { + int unitDuration = units[i].duration * 1000 / unitFileReader.getSampleRate(); + endTime += unitDuration; + Element element = MaryXML.createElement(domElement.getOwnerDocument(), MaryXML.PHONE); + element.setAttribute("d", Integer.toString(unitDuration)); + element.setAttribute("end", Long.toString(endTime)); + element.setAttribute("p", unitNames[i]); + domElement.appendChild(element); + } + + return this.vSynthesizer.synthesizeUsingImposedF0(sourceIndex, targetIndex, aft); + } + + /** + * Synthesize a selected vocalization + * + * @param backchannelNumber + * unit index number + * @param aft + * AudioFileFormat of the output AudioInputStream + * @param domElement + * target 'vocalization' xml element + * @return AudioInputStream output audio + * @throws SynthesisException + * if it can't synthesize vocalization + * @throws IllegalArgumentException + * if given backchannelNumber > no. of available vocalizations + */ + private AudioInputStream synthesizeSelectedVocalization(int backchannelNumber, AudioFileFormat aft, Element domElement) + throws SynthesisException { + + int numberOfBackChannels = unitFileReader.getNumberOfUnits(); + if (backchannelNumber >= numberOfBackChannels) { + throw new IllegalArgumentException("This voice has " + numberOfBackChannels + + " backchannels only. so it doesn't support unit number " + backchannelNumber); + } + + VocalizationUnit bUnit = unitFileReader.getUnit(backchannelNumber); + Unit[] units = bUnit.getUnits(); + String[] unitNames = bUnit.getUnitNames(); + long endTime = 0l; + for (int i = 0; i < units.length; i++) { + int unitDuration = units[i].duration * 1000 / unitFileReader.getSampleRate(); + endTime += unitDuration; + Element element = MaryXML.createElement(domElement.getOwnerDocument(), MaryXML.PHONE); + element.setAttribute("d", Integer.toString(unitDuration)); + element.setAttribute("end", Long.toString(endTime)); + element.setAttribute("p", unitNames[i]); + domElement.appendChild(element); + } + + return this.vSynthesizer.synthesize(backchannelNumber, aft); + } + + /** + * List the possible vocalization names that are available for the given voice. These values can be used in the "name" + * attribute of the vocalization tag. + * + * @return an array of Strings, each string containing one unique vocalization name. + */ + public String[] listAvailableVocalizations() { + FeatureDefinition featureDefinition = vSelector.getFeatureDefinition(); + assert featureDefinition.hasFeature("name"); + int nameIndex = featureDefinition.getFeatureIndex("name"); + return featureDefinition.getPossibleValues(nameIndex); + } +} diff --git a/marytts-purgatory/src/main/java/marytts/vocalizations/VocalizationUnit.java b/marytts-purgatory/src/main/java/marytts/vocalizations/VocalizationUnit.java new file mode 100644 index 0000000000..fa17bf9dea --- /dev/null +++ b/marytts-purgatory/src/main/java/marytts/vocalizations/VocalizationUnit.java @@ -0,0 +1,59 @@ +/** + * Copyright 2006 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.vocalizations; + +import marytts.unitselection.data.Unit; + +/** + * Representation of a unit from a unit database. This gives access to everything that is known about a given unit, including all + * sorts of features and the actual audio data. + * + * @author Sathish pammi + * + */ +public class VocalizationUnit extends marytts.unitselection.data.Unit { + protected Unit[] units; + protected String[] unitNames; + + public VocalizationUnit(long startTime, int duration, int index) { + super(startTime, duration, index); + } + + /** + * Set units + * + * @return + */ + public void setUnits(Unit[] units) { + this.units = units; + } + + public Unit[] getUnits() { + return this.units; + } + + public void setUnitNames(String[] unitNames) { + this.unitNames = unitNames; + } + + public String[] getUnitNames() { + return this.unitNames; + } +} diff --git a/marytts-purgatory/src/main/java/marytts/vocalizations/VocalizationUnitFileReader.java b/marytts-purgatory/src/main/java/marytts/vocalizations/VocalizationUnitFileReader.java new file mode 100644 index 0000000000..990ca7462a --- /dev/null +++ b/marytts-purgatory/src/main/java/marytts/vocalizations/VocalizationUnitFileReader.java @@ -0,0 +1,240 @@ +/** + * Portions Copyright 2006 DFKI GmbH. + * Portions Copyright 2001 Sun Microsystems, Inc. + * Portions Copyright 1999-2001 Language Technologies Institute, + * Carnegie Mellon University. + * All Rights Reserved. Use is subject to license terms. + * + * Permission is hereby granted, free of charge, to use and distribute + * this software and its documentation without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of this work, and to + * permit persons to whom this work is furnished to do so, subject to + * the following conditions: + * + * 1. The code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * 2. Any modifications must be clearly marked as such. + * 3. Original authors' names are not deleted. + * 4. The authors' names are not used to endorse or promote products + * derived from this software without specific prior written + * permission. + * + * DFKI GMBH AND THE CONTRIBUTORS TO THIS WORK DISCLAIM ALL WARRANTIES WITH + * REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL DFKI GMBH NOR THE + * CONTRIBUTORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL + * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR + * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS + * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF + * THIS SOFTWARE. + */ +package marytts.vocalizations; + +import java.io.BufferedInputStream; +import java.io.DataInputStream; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; + +import marytts.exceptions.MaryConfigurationException; +import marytts.unitselection.data.Unit; +import marytts.util.data.MaryHeader; + +/** + * Loads a unit file in memory and provides accessors to the start times and durations. + * + * @author sathish pammi + * + */ +public class VocalizationUnitFileReader { + + private MaryHeader hdr = null; + private int numberOfUnits = 0; + private int sampleRate = 0; + private VocalizationUnit[] backchannelUnits; + + /****************/ + /* CONSTRUCTORS */ + /****************/ + + /** + * Empty constructor; need to call load() separately. + * + * @see #load(String) + */ + public VocalizationUnitFileReader() { + } + + /** + * Create a unit file reader from the given unit file + * + * @param fileName + * the unit file to read + * @throws IOException + * if a problem occurs while reading + */ + public VocalizationUnitFileReader(String fileName) throws IOException, MaryConfigurationException { + load(fileName); + } + + /** + * Load the given unit file + * + * @param fileName + * the unit file to read + * @throws IOException + * if a problem occurs while reading + */ + public void load(String fileName) throws IOException, MaryConfigurationException { + /* Open the file */ + DataInputStream dis = null; + try { + dis = new DataInputStream(new BufferedInputStream(new FileInputStream(fileName))); + } catch (FileNotFoundException e) { + throw new RuntimeException("File [" + fileName + "] was not found."); + } + try { + /* Load the Mary header */ + hdr = new MaryHeader(dis); + if (hdr.getType() != MaryHeader.LISTENERUNITS) { + throw new RuntimeException("File [" + fileName + "] is not a valid Mary Units file."); + } + /* Read the number of units */ + numberOfUnits = dis.readInt(); + // System.out.println("No. of units : "+ numberOfUnits); + if (numberOfUnits < 0) { + throw new RuntimeException("File [" + fileName + "] has a negative number of units. Aborting."); + } + /* Read the sample rate */ + sampleRate = dis.readInt(); + // System.out.println("Samplerate : "+ sampleRate); + if (sampleRate < 0) { + throw new RuntimeException("File [" + fileName + "] has a negative number sample rate. Aborting."); + } + + backchannelUnits = new VocalizationUnit[numberOfUnits]; + + /* Read the start times and durations */ + for (int i = 0; i < numberOfUnits; i++) { + int noOfUnits = dis.readInt(); + // System.out.println("No. of Local Units : "+ noOfUnits); + Unit[] units = new Unit[noOfUnits]; + String[] unitNames = new String[noOfUnits]; + for (int j = 0; j < noOfUnits; j++) { + long startTime = dis.readLong(); + int duration = dis.readInt(); + // System.out.println("Local Unit Data : "+ startTime+" "+ duration+" "+ j); + units[j] = new Unit(startTime, duration, j); + int charArraySize = dis.readInt(); + char[] phoneChar = new char[charArraySize]; + for (int k = 0; k < charArraySize; k++) { + phoneChar[k] = dis.readChar(); + } + unitNames[j] = new String(phoneChar); + } + long startBCTime = units[0].startTime; + int bcDuration = (((int) units[noOfUnits - 1].startTime + units[noOfUnits - 1].duration) - (int) units[0].startTime); + backchannelUnits[i] = new VocalizationUnit(startBCTime, bcDuration, i); + backchannelUnits[i].setUnits(units); + backchannelUnits[i].setUnitNames(unitNames); + // System.out.println("BC UNIT START:"+backchannelUnits[i].getStart()); + // System.out.println("BC UNIT Duration:"+backchannelUnits[i].getDuration()); + } + } catch (IOException e) { + throw new RuntimeException("Reading the Mary header from file [" + fileName + "] failed.", e); + } + + } + + /*****************/ + /* OTHER METHODS */ + /*****************/ + + /** + * Get the number of units in the file. + * + * @return The number of units. + */ + public int getNumberOfUnits() { + return (numberOfUnits); + } + + /** + * Get the sample rate of the file. + * + * @return The sample rate, in Hz. + */ + public int getSampleRate() { + return (sampleRate); + } + + /** + * Return the unit number i. + * + * @param i + * The index of the considered unit. + * @return The considered unit. + */ + public VocalizationUnit getUnit(int i) { + return backchannelUnits[i]; + } + + /** + * Return an array of units from their indexes. + * + * @param i + * The indexes of the considered units. + * @return The array of considered units. + */ + public VocalizationUnit[] getUnit(int[] i) { + VocalizationUnit[] ret = new VocalizationUnit[i.length]; + for (int k = 0; k < i.length; k++) { + ret[k] = getUnit(i[k]); + } + return (ret); + } + + /** + * Return the unit following the given unit in the original database. + * + * @param u + * a unit + * @return the next unit in the database, or null if there is no such unit. + */ + public VocalizationUnit getNextUnit(VocalizationUnit u) { + if (u == null || u.index >= backchannelUnits.length - 1 || u.index < 0) + return null; + return backchannelUnits[u.index + 1]; + } + + /** + * Return the unit preceding the given unit in the original database. + * + * @param u + * a unit + * @return the previous unit in the database, or null if there is no such unit. + */ + public VocalizationUnit getPreviousUnit(VocalizationUnit u) { + if (u == null || u.index >= backchannelUnits.length || u.index <= 0) + return null; + return backchannelUnits[u.index - 1]; + } + + /** + * Determine whether the unit number i is an "edge" unit, i.e. a unit marking the start or the end of an utterance. + * + * @param i + * The index of the considered unit. + * @return true if the unit is an edge unit in the unit file, false otherwise + */ + public boolean isEdgeUnit(int i) { + return backchannelUnits[i].isEdgeUnit(); + } + + public static void main(String[] args) throws Exception { + String fileName = "/home/sathish/Work/dfki399/backchannel/mary_files/BCCphoneUnits.mry"; + VocalizationUnitFileReader bcUfr = new VocalizationUnitFileReader(); + bcUfr.load(fileName); + } +} From e06d5259d5111264827e63c2ad5a6a6c875150de Mon Sep 17 00:00:00 2001 From: Tristan Hamilton Date: Mon, 9 Feb 2015 13:41:51 +0100 Subject: [PATCH 26/31] vocalisation builder packages sent to purgatory purgatory reorganised into two sub folders, runtime and builder - copying their folder hierarchies --- .../vocalizations/HNMFeatureFileWriter.java | 232 +++++++++ .../MLSAFeatureFileComputer.java | 397 ++++++++++++++ .../vocalizations/MLSAFeatureFileWriter.java | 257 ++++++++++ .../SnackF0ContourExtractor.java | 239 +++++++++ .../VocalizationF0PolyFeatureFileWriter.java | 484 ++++++++++++++++++ .../VocalizationF0PolynomialInspector.java | 449 ++++++++++++++++ .../VocalizationFeatureFileWriter.java | 278 ++++++++++ .../VocalizationIntonationWriter.java | 435 ++++++++++++++++ .../VocalizationPitchmarker.java | 160 ++++++ .../VocalizationTimelineMaker.java | 233 +++++++++ .../VocalizationUnitfileWriter.java | 319 ++++++++++++ .../VocalisationLabelInspector.java | 252 +++++++++ .../VocalizationAnnotationReader.java | 128 +++++ .../java/marytts/modules/CARTF0Modeller.java | 0 .../InterpolatingSynthesizer.java | 0 .../interpolation/InterpolatingVoice.java | 0 .../VocalizationFFRTargetCostFunction.java | 0 .../FDPSOLASynthesisTechnology.java | 0 .../vocalizations/HNMFeatureFileReader.java | 0 .../vocalizations/HNMSynthesisTechnology.java | 0 .../vocalizations/KMeansClusterer.java | 0 .../vocalizations/MLSAFeatureFileReader.java | 0 .../MLSASynthesisTechnology.java | 0 .../vocalizations/SourceTargetPair.java | 0 .../vocalizations/VocalizationCandidate.java | 0 .../VocalizationFeatureFileReader.java | 0 .../VocalizationIntonationReader.java | 0 .../vocalizations/VocalizationSelector.java | 0 .../VocalizationSynthesisTechnology.java | 0 .../VocalizationSynthesizer.java | 0 .../vocalizations/VocalizationUnit.java | 0 .../VocalizationUnitFileReader.java | 0 32 files changed, 3863 insertions(+) create mode 100644 marytts-purgatory/marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/HNMFeatureFileWriter.java create mode 100644 marytts-purgatory/marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/MLSAFeatureFileComputer.java create mode 100644 marytts-purgatory/marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/MLSAFeatureFileWriter.java create mode 100644 marytts-purgatory/marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/SnackF0ContourExtractor.java create mode 100644 marytts-purgatory/marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/VocalizationF0PolyFeatureFileWriter.java create mode 100644 marytts-purgatory/marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/VocalizationF0PolynomialInspector.java create mode 100644 marytts-purgatory/marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/VocalizationFeatureFileWriter.java create mode 100644 marytts-purgatory/marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/VocalizationIntonationWriter.java create mode 100644 marytts-purgatory/marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/VocalizationPitchmarker.java create mode 100644 marytts-purgatory/marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/VocalizationTimelineMaker.java create mode 100644 marytts-purgatory/marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/VocalizationUnitfileWriter.java create mode 100644 marytts-purgatory/marytts-builder/src/main/java/marytts/vocalizations/VocalisationLabelInspector.java create mode 100644 marytts-purgatory/marytts-builder/src/main/java/marytts/vocalizations/VocalizationAnnotationReader.java rename marytts-purgatory/{ => marytts-runtime}/src/main/java/marytts/modules/CARTF0Modeller.java (100%) rename marytts-purgatory/{ => marytts-runtime}/src/main/java/marytts/unitselection/interpolation/InterpolatingSynthesizer.java (100%) rename marytts-purgatory/{ => marytts-runtime}/src/main/java/marytts/unitselection/interpolation/InterpolatingVoice.java (100%) rename marytts-purgatory/{ => marytts-runtime}/src/main/java/marytts/unitselection/select/VocalizationFFRTargetCostFunction.java (100%) rename marytts-purgatory/{ => marytts-runtime}/src/main/java/marytts/vocalizations/FDPSOLASynthesisTechnology.java (100%) rename marytts-purgatory/{ => marytts-runtime}/src/main/java/marytts/vocalizations/HNMFeatureFileReader.java (100%) rename marytts-purgatory/{ => marytts-runtime}/src/main/java/marytts/vocalizations/HNMSynthesisTechnology.java (100%) rename marytts-purgatory/{ => marytts-runtime}/src/main/java/marytts/vocalizations/KMeansClusterer.java (100%) rename marytts-purgatory/{ => marytts-runtime}/src/main/java/marytts/vocalizations/MLSAFeatureFileReader.java (100%) rename marytts-purgatory/{ => marytts-runtime}/src/main/java/marytts/vocalizations/MLSASynthesisTechnology.java (100%) rename marytts-purgatory/{ => marytts-runtime}/src/main/java/marytts/vocalizations/SourceTargetPair.java (100%) rename marytts-purgatory/{ => marytts-runtime}/src/main/java/marytts/vocalizations/VocalizationCandidate.java (100%) rename marytts-purgatory/{ => marytts-runtime}/src/main/java/marytts/vocalizations/VocalizationFeatureFileReader.java (100%) rename marytts-purgatory/{ => marytts-runtime}/src/main/java/marytts/vocalizations/VocalizationIntonationReader.java (100%) rename marytts-purgatory/{ => marytts-runtime}/src/main/java/marytts/vocalizations/VocalizationSelector.java (100%) rename marytts-purgatory/{ => marytts-runtime}/src/main/java/marytts/vocalizations/VocalizationSynthesisTechnology.java (100%) rename marytts-purgatory/{ => marytts-runtime}/src/main/java/marytts/vocalizations/VocalizationSynthesizer.java (100%) rename marytts-purgatory/{ => marytts-runtime}/src/main/java/marytts/vocalizations/VocalizationUnit.java (100%) rename marytts-purgatory/{ => marytts-runtime}/src/main/java/marytts/vocalizations/VocalizationUnitFileReader.java (100%) diff --git a/marytts-purgatory/marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/HNMFeatureFileWriter.java b/marytts-purgatory/marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/HNMFeatureFileWriter.java new file mode 100644 index 0000000000..86a50ec887 --- /dev/null +++ b/marytts-purgatory/marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/HNMFeatureFileWriter.java @@ -0,0 +1,232 @@ +/** + * Copyright 2010 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.tools.voiceimport.vocalizations; + +import java.io.DataOutput; +import java.io.DataOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.SortedMap; +import java.util.TreeMap; + +import javax.sound.sampled.AudioInputStream; +import javax.sound.sampled.AudioSystem; +import javax.sound.sampled.UnsupportedAudioFileException; + +import marytts.exceptions.MaryConfigurationException; +import marytts.signalproc.analysis.F0TrackerAutocorrelationHeuristic; +import marytts.signalproc.analysis.PitchFileHeader; +import marytts.signalproc.analysis.PitchReaderWriter; +import marytts.signalproc.analysis.RegularizedCepstrumEstimator; +import marytts.signalproc.sinusoidal.hntm.analysis.HntmAnalyzer; +import marytts.signalproc.sinusoidal.hntm.analysis.HntmAnalyzerParams; +import marytts.signalproc.sinusoidal.hntm.analysis.HntmSpeechSignal; +import marytts.signalproc.sinusoidal.hntm.synthesis.HntmSynthesizerParams; +import marytts.tools.voiceimport.DatabaseLayout; +import marytts.tools.voiceimport.VoiceImportComponent; +import marytts.util.data.MaryHeader; +import marytts.util.data.audio.AudioDoubleDataSource; +import marytts.util.io.BasenameList; +import marytts.util.math.MathUtils; +import marytts.vocalizations.HNMFeatureFileReader; +import marytts.vocalizations.VocalizationUnitFileReader; + +/** + * A component to extract and write HNM features of all vocalizations into a single file + * + * @author sathish + * + */ +public class HNMFeatureFileWriter extends VoiceImportComponent { + + private String waveExt = ".wav"; + private String ptcExt = ".ptc"; + private String hnmAnalysisFileExt = ".ana"; + private int progress = 0; + protected DatabaseLayout db = null; + protected BasenameList bnlVocalizations; + + private final String WAVEDIR = getName() + ".vocalizationWaveDir"; + private final String HNMANADIR = getName() + ".hnmAnalysisDir"; + private final String OUTHNMFILE = getName() + ".hnmAnalysisTimelineFile"; + private final String UNITFILE = getName() + ".unitFile"; + + private HntmSynthesizerParams synthesisParamsBeforeNoiseAnalysis; + private HntmAnalyzerParams analysisParams; + private PitchFileHeader f0Params; + + @Override + public SortedMap getDefaultProps(DatabaseLayout db) { + this.db = db; + if (props == null) { + props = new TreeMap(); + props.put(UNITFILE, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "files" + File.separator + + "vocalization_units" + db.getProp(db.MARYEXT)); + props.put(HNMANADIR, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "hna" + File.separator); + props.put(WAVEDIR, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "wav"); + props.put(OUTHNMFILE, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "files" + File.separator + + "vocalization_hnm_analysis" + db.getProp(db.MARYEXT)); + } + return props; + } + + @Override + protected void setupHelp() { + if (props2Help == null) { + props2Help = new TreeMap(); + props2Help.put(UNITFILE, "unit file representing all vocalizations"); + props2Help.put(HNMANADIR, "HNM features directory"); + props2Help.put(WAVEDIR, "vocalization wave files directory"); + props2Help.put(OUTHNMFILE, "a single file to write all HNM features"); + } + } + + @Override + public boolean compute() throws UnsupportedAudioFileException, IOException, MaryConfigurationException { + + VocalizationUnitFileReader listenerUnits = new VocalizationUnitFileReader(getProp(UNITFILE)); + + // sanity checker + if (listenerUnits.getNumberOfUnits() != bnlVocalizations.getLength()) { + throw new MaryConfigurationException( + "The number of vocalizations given in basename list and number of units in unit file should be matched"); + } + + F0TrackerAutocorrelationHeuristic pitchDetector = new F0TrackerAutocorrelationHeuristic(f0Params); + HntmAnalyzer ha = new HntmAnalyzer(); + DataOutputStream outStream = new DataOutputStream(new FileOutputStream(new File(getProp(OUTHNMFILE)))); + writeHeaderTo(outStream); + outStream.writeInt(listenerUnits.getNumberOfUnits()); + + for (int i = 0; i < bnlVocalizations.getLength(); i++) { + progress = i * 100 / bnlVocalizations.getLength(); + String wavFile = getProp(WAVEDIR) + File.separator + bnlVocalizations.getName(i) + waveExt; + String pitchFile = getProp(HNMANADIR) + File.separator + bnlVocalizations.getName(i) + ptcExt; + String analysisResultsFile = getProp(HNMANADIR) + File.separator + bnlVocalizations.getName(i) + hnmAnalysisFileExt; + PitchReaderWriter f0 = pitchDetector.pitchAnalyzeWavFile(wavFile, pitchFile); + AudioInputStream inputAudio = AudioSystem.getAudioInputStream(new File(wavFile)); + int samplingRate = (int) inputAudio.getFormat().getSampleRate(); + AudioDoubleDataSource signal = new AudioDoubleDataSource(inputAudio); + double[] x = signal.getAllData(); + x = MathUtils.multiply(x, 32768.0); + HntmSpeechSignal hnmSignal = ha.analyze(x, samplingRate, f0, null, analysisParams, + synthesisParamsBeforeNoiseAnalysis, analysisResultsFile); + hnmSignal.write(outStream); + } + outStream.close(); + + HNMFeatureFileReader tester = new HNMFeatureFileReader(getProp(OUTHNMFILE)); + int unitsOnDisk = tester.getNumberOfUnits(); + if (unitsOnDisk == listenerUnits.getNumberOfUnits()) { + System.out.println("Can read right number of units"); + return true; + } else { + System.out.println("Read wrong number of units: " + unitsOnDisk); + return false; + } + } + + /** + * Initialize this component + */ + @Override + protected void initialiseComp() throws Exception { + + createDirectoryifNotExists(getProp(HNMANADIR)); + + try { + String basenameFile = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "basenames.lst"; + if ((new File(basenameFile)).exists()) { + System.out.println("Loading basenames of vocalizations from '" + basenameFile + "' list..."); + bnlVocalizations = new BasenameList(basenameFile); + System.out.println("Found " + bnlVocalizations.getLength() + " vocalizations in basename list"); + } else { + String vocalWavDir = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "wav"; + System.out.println("Loading basenames of vocalizations from '" + vocalWavDir + "' directory..."); + bnlVocalizations = new BasenameList(vocalWavDir, ".wav"); + System.out.println("Found " + bnlVocalizations.getLength() + " vocalizations in " + vocalWavDir + " directory"); + } + } catch (IOException e) { + throw new MaryConfigurationException("Problem with basename list " + e); + } + f0Params = new PitchFileHeader(); + analysisParams = new HntmAnalyzerParams(); + + analysisParams.harmonicModel = HntmAnalyzerParams.HARMONICS_PLUS_NOISE; + analysisParams.noiseModel = HntmAnalyzerParams.WAVEFORM; + analysisParams.useHarmonicAmplitudesDirectly = true; + analysisParams.harmonicSynthesisMethodBeforeNoiseAnalysis = HntmSynthesizerParams.LINEAR_PHASE_INTERPOLATION; + analysisParams.regularizedCepstrumWarpingMethod = RegularizedCepstrumEstimator.REGULARIZED_CEPSTRUM_WITH_POST_MEL_WARPING; + + synthesisParamsBeforeNoiseAnalysis = new HntmSynthesizerParams(); + synthesisParamsBeforeNoiseAnalysis.harmonicPartSynthesisMethod = HntmSynthesizerParams.LINEAR_PHASE_INTERPOLATION; + } + + /** + * Create new directory if the directory doesn't exist + * + * @param dirName + * @throws Exception + */ + private void createDirectoryifNotExists(String dirName) throws Exception { + if (!(new File(dirName)).exists()) { + System.out.println(dirName + " directory does not exist; "); + if (!(new File(dirName)).mkdirs()) { + throw new Exception("Could not create directory " + dirName); + } + System.out.println("Created successfully.\n"); + } + } + + /** + * Write the header of this feature file to the given DataOutput + * + * @param out + * @throws IOException + */ + protected void writeHeaderTo(DataOutput out) throws IOException { + new MaryHeader(MaryHeader.LISTENERFEATS).writeTo(out); + } + + /** + * Return this voice import component name + */ + @Override + public String getName() { + return "HNMFeatureFileWriter"; + } + + /** + * Return the progress of this component + */ + @Override + public int getProgress() { + return this.progress; + } + + /** + * @param args + */ + public static void main(String[] args) { + // TODO Auto-generated method stub + + } +} diff --git a/marytts-purgatory/marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/MLSAFeatureFileComputer.java b/marytts-purgatory/marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/MLSAFeatureFileComputer.java new file mode 100644 index 0000000000..bb11c4658c --- /dev/null +++ b/marytts-purgatory/marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/MLSAFeatureFileComputer.java @@ -0,0 +1,397 @@ +/** + * Copyright 2010 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.tools.voiceimport.vocalizations; + +import java.io.File; +import java.io.IOException; +import java.util.SortedMap; +import java.util.TreeMap; + +import marytts.exceptions.ExecutionException; +import marytts.exceptions.MaryConfigurationException; +import marytts.tools.voiceimport.DatabaseLayout; +import marytts.tools.voiceimport.VoiceImportComponent; +import marytts.util.io.BasenameList; +import marytts.util.io.FileUtils; +import marytts.util.io.General; + +/** + * MLSA feature files extractor for vocalizations. It extracts mgc features, strengths and logf0 features + * + * @author sathish + * + */ +public class MLSAFeatureFileComputer extends VoiceImportComponent { + + private String waveExt = ".wav"; + private String rawExt = ".raw"; + private String lf0Ext = ".lf0"; + private String strExt = ".str"; + private String mgcExt = ".mgc"; + private int progress = -1; + + // Default LF0 parameters + private int SAMPFREQ = 16000; + private int FRAMESHIFT = 80; + private int LOWERF0_VALUE = 100; + private int UPPERF0_VALUE = 500; + // Default MGC parameters + private int FRAMELEN = 400; + private int FFTLEN = 512; + private float FREQWARP = 0.42f; + private int MGCORDER = 24; // MGCORDER is actually 24 plus 1 include MGC[0] + // Default STR parameters + private int STRORDER = 5; + + protected DatabaseLayout db = null; + protected BasenameList bnlVocalizations; + + private final String WAVEDIR = getName() + ".vocalizationWaveDir"; + private final String MLSADIR = getName() + ".vocalizationMLSAFilesDir"; + private final String RAWDIR = getName() + ".rawFilesDir"; + private final String SCRIPTSDIR = getName() + ".scriptsDir"; + private final String TCLCOMMAND = getName() + ".tclsh-commandlinePath"; + private final String SOXCOMMAND = getName() + ".sox-commandlinePath"; + private final String SPTKPATH = getName() + ".SPTK-Path"; + private final String LOWERLF0 = getName() + ".LowerF0Value"; + private final String UPPERLF0 = getName() + ".UpperF0Value"; + + @Override + public SortedMap getDefaultProps(DatabaseLayout db) { + this.db = db; + if (props == null) { + props = new TreeMap(); + props.put(WAVEDIR, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "wav"); + props.put(SCRIPTSDIR, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "scripts"); + String mlsaDir = db.getProp(db.VOCALIZATIONSDIR) + "mlsa"; + props.put(MLSADIR, mlsaDir); + props.put(RAWDIR, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "raw"); + props.put(TCLCOMMAND, "/usr/bin/tclsh"); + props.put(SOXCOMMAND, "/usr/bin/sox"); + props.put(SPTKPATH, "/usr/bin/"); + props.put(LOWERLF0, LOWERF0_VALUE + ""); + props.put(UPPERLF0, UPPERF0_VALUE + ""); + } + return props; + } + + @Override + protected void setupHelp() { + if (props2Help == null) { + props2Help = new TreeMap(); + props2Help.put(WAVEDIR, "directory that contains vocalization wave files "); + props2Help.put(SCRIPTSDIR, "directory that contains external scripts used to compute MGC, LF0 and MGC features"); + props2Help.put(MLSADIR, "mlsa features directory"); + props2Help.put(RAWDIR, "raw files directory"); + props2Help.put(TCLCOMMAND, "tcl executable command"); + props2Help.put(SOXCOMMAND, "sox executable command"); + props2Help.put(SPTKPATH, "Path that contains SPTK executables"); + props2Help.put(LOWERLF0, "lowest pitch value specification"); + props2Help.put(UPPERLF0, "highest pitch value specification"); + } + + } + + /** + * compute logf0, mgc, strength features + */ + @Override + public boolean compute() throws Exception { + + copyFilesandScripts(); + convertWAVE2RAW(); + computeLF0Features(); + computeMGCFeatures(); + computeSTRFeatures(); + + return true; + } + + /** + * Initialize this component + * + * @throws MaryConfigurationException + * if there is problem with basename list + */ + @Override + protected void initialiseComp() throws Exception { + + createDirectoryifNotExists(getProp(MLSADIR)); + createDirectoryifNotExists(getProp(RAWDIR)); + + try { + String basenameFile = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "basenames.lst"; + if ((new File(basenameFile)).exists()) { + System.out.println("Loading basenames of vocalizations from '" + basenameFile + "' list..."); + bnlVocalizations = new BasenameList(basenameFile); + System.out.println("Found " + bnlVocalizations.getLength() + " vocalizations in basename list"); + } else { + String vocalWavDir = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "wav"; + System.out.println("Loading basenames of vocalizations from '" + vocalWavDir + "' directory..."); + bnlVocalizations = new BasenameList(vocalWavDir, ".wav"); + System.out.println("Found " + bnlVocalizations.getLength() + " vocalizations in " + vocalWavDir + " directory"); + } + } catch (IOException e) { + throw new MaryConfigurationException("Problem with basename list " + e); + } + } + + /** + * copy required scripts and files for third-party software execution + * + * @throws Exception + * if it can't copy files from source location + */ + private void copyFilesandScripts() throws Exception { + + String sep = System.getProperty("file.separator"); + + createDirectoryifNotExists(getProp(SCRIPTSDIR)); + createDirectoryifNotExists(db.getProp(db.ROOTDIR) + sep + "filters"); + + String logF0ShellScript = getProp(this.SCRIPTSDIR) + sep + "get_f0.sh"; + String logF0TCLScript = getProp(this.SCRIPTSDIR) + sep + "get_f0.tcl"; + String mgcShellScript = getProp(this.SCRIPTSDIR) + sep + "get_mgc.sh"; + String strShellScript = getProp(this.SCRIPTSDIR) + sep + "get_str.sh"; + String strTCLScript = getProp(this.SCRIPTSDIR) + sep + "get_str.tcl"; + String filterFile = db.getProp(db.ROOTDIR) + sep + "filters" + sep + "mix_excitation_filters.txt"; + String sourceDir = db.getProp(db.MARYBASE) + sep + "lib" + sep + "external" + sep + "vocalizations"; + + if (!FileUtils.exists(logF0ShellScript)) { + String sourceScript = sourceDir + sep + "scripts" + sep + "get_f0.sh"; + FileUtils.copy(sourceScript, logF0ShellScript); + (new File(logF0ShellScript)).setExecutable(true); + } + + if (!FileUtils.exists(logF0TCLScript)) { + String sourceScript = sourceDir + sep + "scripts" + sep + "get_f0.tcl"; + FileUtils.copy(sourceScript, logF0TCLScript); + (new File(logF0TCLScript)).setExecutable(true); + } + + if (!FileUtils.exists(strShellScript)) { + String sourceScript = sourceDir + sep + "scripts" + sep + "get_str.sh"; + FileUtils.copy(sourceScript, strShellScript); + (new File(strShellScript)).setExecutable(true); + } + + if (!FileUtils.exists(strTCLScript)) { + String sourceScript = sourceDir + sep + "scripts" + sep + "get_str.tcl"; + FileUtils.copy(sourceScript, strTCLScript); + (new File(strTCLScript)).setExecutable(true); + } + + if (!FileUtils.exists(mgcShellScript)) { + String sourceScript = sourceDir + sep + "scripts" + sep + "get_mgc.sh"; + FileUtils.copy(sourceScript, mgcShellScript); + (new File(mgcShellScript)).setExecutable(true); + } + + if (!FileUtils.exists(filterFile)) { + String sourceScript = sourceDir + sep + "filters" + sep + "mix_excitation_filters.txt"; + FileUtils.copy(sourceScript, filterFile); + } + } + + /** + * Convert all WAV files to RAW files to support SPTK + * + * @throws IOException + * if can't run command + * @throws ExecutionException + * if commandline throws an error + */ + private void convertWAVE2RAW() throws Exception { + + for (int i = 0; i < bnlVocalizations.getLength(); i++) { + String waveFile = getProp(WAVEDIR) + File.separator + bnlVocalizations.getName(i) + waveExt; + String rawFile = getProp(RAWDIR) + File.separator + bnlVocalizations.getName(i) + rawExt; + String command = getProp(SOXCOMMAND) + " " + waveFile + " " + rawFile; + // System.out.println( "Command: " + command ); + + try { + General.launchProc(command, "MLSAFeatureFileComputer", bnlVocalizations.getName(i)); + } catch (Exception e) { + throw new ExecutionException("\nCommand failed : " + command + "\n" + e); + } + + File rawEmptyFile = new File(rawFile); + if (rawEmptyFile.length() <= 0) { // delete all empty files + rawEmptyFile.delete(); + } + System.out.println("Creating " + bnlVocalizations.getName(i) + " RAW file"); + + File rawNFile = new File(rawFile); + if (!rawNFile.exists()) { + throw new ExecutionException("The following command failed: \n " + command + "\n"); + } + } + } + + /** + * Compute LF0 features for all RAW files using Snack and SPTK + * + * @throws IOException + * if can't run command + * @throws ExecutionException + * if commandline throws an error + * @throws ExecutionException + * if there is no output file or output file is empty + */ + private void computeLF0Features() throws Exception { + String bcCommand = "/usr/bin/bc"; + + for (int i = 0; i < bnlVocalizations.getLength(); i++) { + String rawFile = getProp(RAWDIR) + File.separator + bnlVocalizations.getName(i) + rawExt; + String lf0File = getProp(MLSADIR) + File.separator + bnlVocalizations.getName(i) + lf0Ext; + String command = getProp(this.SCRIPTSDIR) + File.separator + "get_f0.sh " + bcCommand + " " + getProp(SPTKPATH) + " " + + getProp(TCLCOMMAND) + " " + getProp(this.SCRIPTSDIR) + " " + rawFile + " " + lf0File + " " + SAMPFREQ + " " + + FRAMESHIFT + " " + getProp(LOWERLF0) + " " + getProp(UPPERLF0); + try { + General.launchProc(command, "MLSAFeatureFileComputer", bnlVocalizations.getName(i)); + } catch (Exception e) { + throw new ExecutionException("\nCommand failed : " + command + "\n" + e); + } + File lf0EmptyFile = new File(lf0File); + if (lf0EmptyFile.length() <= 0) { // delete all empty files + lf0EmptyFile.delete(); + } + // System.out.println( "Command: " + command ); + File lf0NFile = new File(lf0File); + if (!lf0NFile.exists()) { + throw new ExecutionException("The following command failed: \n " + command + "\n"); + } + System.out.println("Computed LF0 features for " + bnlVocalizations.getName(i)); + } + } + + /** + * Compute MGC features for all RAW files using SPTK + * + * @throws IOException + * if can't run command + * @throws ExecutionException + * if commandline throws an error + * @throws ExecutionException + * if there is no output file or output file is empty + */ + private void computeMGCFeatures() throws Exception { + + for (int i = 0; i < bnlVocalizations.getLength(); i++) { + String rawFile = getProp(RAWDIR) + File.separator + bnlVocalizations.getName(i) + rawExt; + String mgcFile = getProp(MLSADIR) + File.separator + bnlVocalizations.getName(i) + mgcExt; + String command = getProp(this.SCRIPTSDIR) + File.separator + "get_mgc.sh " + getProp(SPTKPATH) + " " + FRAMELEN + " " + + FRAMESHIFT + " " + FFTLEN + " " + FREQWARP + " " + MGCORDER + " " + rawFile + " " + mgcFile; + try { + General.launchProc(command, "MLSAFeatureFileComputer", bnlVocalizations.getName(i)); + } catch (Exception e) { + throw new ExecutionException("\nCommand failed : " + command + "\n" + e); + } + File lf0EmptyFile = new File(mgcFile); + if (lf0EmptyFile.length() <= 0) { // delete all empty files + lf0EmptyFile.delete(); + } + // System.out.println( "Command: " + command ); + + File lf0NFile = new File(mgcFile); + if (!lf0NFile.exists()) { + throw new ExecutionException("The following command failed: \n " + command + "\n"); + } + System.out.println("Computed MGC features for " + bnlVocalizations.getName(i)); + } + } + + /** + * Compute STRENGTH features for all RAW files using SPTK + * + * @throws IOException + * if can't run command + * @throws ExecutionException + * if commandline throws an error + * @throws ExecutionException + * if there is no output file or output file is empty + */ + private void computeSTRFeatures() throws Exception { + String bcCommand = "/usr/bin/bc"; + + for (int i = 0; i < bnlVocalizations.getLength(); i++) { + String rawFile = getProp(RAWDIR) + File.separator + bnlVocalizations.getName(i) + rawExt; + String strFile = getProp(MLSADIR) + File.separator + bnlVocalizations.getName(i) + strExt; + String command = getProp(this.SCRIPTSDIR) + File.separator + "get_str.sh " + bcCommand + " " + getProp(SPTKPATH) + + " " + getProp(TCLCOMMAND) + " " + getProp(this.SCRIPTSDIR) + " " + SAMPFREQ + " " + FRAMESHIFT + " " + + getProp(LOWERLF0) + " " + getProp(UPPERLF0) + " " + STRORDER + " " + rawFile + " " + strFile + " "; + try { + General.launchProc(command, "MLSAFeatureFileComputer", bnlVocalizations.getName(i)); + } catch (Exception e) { + throw new ExecutionException("\nCommand failed : " + command + "\n" + e); + } + File lf0EmptyFile = new File(strFile); + if (lf0EmptyFile.length() <= 0) { // delete all empty files + lf0EmptyFile.delete(); + } + // System.out.println( "Command: " + command ); + File lf0NFile = new File(strFile); + if (!lf0NFile.exists()) { + throw new ExecutionException("The following command failed: \n " + command + "\n"); + } + System.out.println("Computed Strength features for " + bnlVocalizations.getName(i)); + } + } + + /** + * Create new directory if the directory doesn't exist + * + * @param dirName + * @throws Exception + */ + private void createDirectoryifNotExists(String dirName) throws Exception { + if (!(new File(dirName)).exists()) { + System.out.println(dirName + " directory does not exist; "); + if (!(new File(dirName)).mkdirs()) { + throw new Exception("Could not create directory " + dirName); + } + System.out.println("Created successfully.\n"); + } + } + + /** + * Return this voice import component name + */ + @Override + public String getName() { + return "MLSAFeatureFileComputer"; + } + + /** + * Return the progress of this component + */ + @Override + public int getProgress() { + return this.progress; + } + + /** + * @param args + */ + public static void main(String[] args) { + // TODO Auto-generated method stub + + } +} diff --git a/marytts-purgatory/marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/MLSAFeatureFileWriter.java b/marytts-purgatory/marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/MLSAFeatureFileWriter.java new file mode 100644 index 0000000000..7c16e1ec77 --- /dev/null +++ b/marytts-purgatory/marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/MLSAFeatureFileWriter.java @@ -0,0 +1,257 @@ +/** + * Copyright 2010 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.tools.voiceimport.vocalizations; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.DataOutput; +import java.io.DataOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.SortedMap; +import java.util.TreeMap; + +import marytts.exceptions.MaryConfigurationException; +import marytts.tools.voiceimport.DatabaseLayout; +import marytts.tools.voiceimport.VoiceImportComponent; +import marytts.util.data.MaryHeader; +import marytts.util.io.BasenameList; +import marytts.util.io.LEDataInputStream; +import marytts.util.math.MathUtils; +import marytts.vocalizations.MLSAFeatureFileReader; +import marytts.vocalizations.VocalizationUnitFileReader; + +/** + * A component to write all MLSA features (logf0, mgc and strengths) into a single file + * + * @author sathish + * + */ +public class MLSAFeatureFileWriter extends VoiceImportComponent { + + private String lf0Ext = ".lf0"; + private String strExt = ".str"; + private String mgcExt = ".mgc"; + private int progress = 0; + protected VocalizationUnitFileReader listenerUnits; + + private int MGCORDER = 25; // MGCORDER is actually 24 plus 1 include MGC[0] + private int STRORDER = 5; + private int LF0ORDER = 1; + + protected DatabaseLayout db = null; + protected BasenameList bnlVocalizations; + + public final String MLSADIR = getName() + ".vocalizationMLSAFilesDir"; + public final String UNITFILE = getName() + ".unitFile"; + public final String OUTMLSAFILE = getName() + ".mlsaOutputFile"; + + @Override + public SortedMap getDefaultProps(DatabaseLayout db) { + this.db = db; + if (props == null) { + props = new TreeMap(); + String mlsaDir = db.getProp(db.VOCALIZATIONSDIR) + "mlsa"; + props.put(MLSADIR, mlsaDir); + props.put(UNITFILE, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "files" + File.separator + + "vocalization_units" + db.getProp(db.MARYEXT)); + props.put(OUTMLSAFILE, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "files" + File.separator + + "vocalization_mlsa_features" + db.getProp(db.MARYEXT)); + } + return props; + } + + @Override + protected void setupHelp() { + + if (props2Help == null) { + props2Help = new TreeMap(); + props2Help.put(MLSADIR, "mlsa features directory"); + props2Help.put(UNITFILE, "unit file representing all vocalizations"); + props2Help.put(OUTMLSAFILE, "a single file to write all mlsa features"); + } + + } + + @Override + public boolean compute() throws Exception { + + listenerUnits = new VocalizationUnitFileReader(getProp(UNITFILE)); + + // write features into timeline file + DataOutputStream out = new DataOutputStream( + new BufferedOutputStream(new FileOutputStream(new File(getProp(OUTMLSAFILE))))); + writeHeaderTo(out); + writeUnitFeaturesTo(out); + out.close(); + + MLSAFeatureFileReader tester = new MLSAFeatureFileReader(getProp(OUTMLSAFILE)); + int unitsOnDisk = tester.getNumberOfUnits(); + if (unitsOnDisk == listenerUnits.getNumberOfUnits()) { + System.out.println("Can read right number of units"); + return true; + } else { + System.out.println("Read wrong number of units: " + unitsOnDisk); + return false; + } + } + + /** + * write all features into a single file + * + * @param out + * DataOutputStream + * @throws IOException + * if it can't write data into DataOutputStream + */ + private void writeUnitFeaturesTo(DataOutputStream out) throws IOException { + + int numUnits = listenerUnits.getNumberOfUnits(); + out.writeInt(numUnits); // write number of units + out.writeInt(this.LF0ORDER); // write lf0 order + out.writeInt(this.MGCORDER); // write MGC order + out.writeInt(this.STRORDER); // write STR order + + assert bnlVocalizations.getLength() == numUnits : "number of units and size of basename list should be same"; + + for (int i = 0; i < bnlVocalizations.getLength(); i++) { + + String mgcFile = getProp(MLSADIR) + File.separator + bnlVocalizations.getName(i) + mgcExt; + String lf0File = getProp(MLSADIR) + File.separator + bnlVocalizations.getName(i) + lf0Ext; + String strFile = getProp(MLSADIR) + File.separator + bnlVocalizations.getName(i) + strExt; + + LEDataInputStream lf0Data = new LEDataInputStream(new BufferedInputStream(new FileInputStream(lf0File))); + LEDataInputStream mgcData = new LEDataInputStream(new BufferedInputStream(new FileInputStream(mgcFile))); + LEDataInputStream strData = new LEDataInputStream(new BufferedInputStream(new FileInputStream(strFile))); + + int mgcSize = (int) ((new File(mgcFile)).length() / 4); // 4 bytes for float + int lf0Size = (int) ((new File(lf0File)).length() / 4); // 4 bytes for float + int strSize = (int) ((new File(strFile)).length() / 4); // 4 bytes for float + + float sizes[] = new float[3]; + int n = 0; + sizes[n++] = mgcSize / (this.MGCORDER); + sizes[n++] = lf0Size; + sizes[n++] = strSize / this.STRORDER; + int numberOfFrames = (int) MathUtils.getMin(sizes); + out.writeInt(numberOfFrames); // number of frames in this vocalization + + // first write LF0 data + int numberOfLF0Frames = numberOfFrames; + out.writeInt(numberOfLF0Frames); // number of LF0 frames + for (int j = 0; j < numberOfLF0Frames; j++) { + out.writeFloat(lf0Data.readFloat()); + } + + // second write MGC data + int numberOfMGCFrames = numberOfFrames * (this.MGCORDER); + out.writeInt(numberOfMGCFrames); // number of MGC frames + for (int j = 0; j < numberOfMGCFrames; j++) { + out.writeFloat(mgcData.readFloat()); + } + + // Third write STR data + int numberOfSTRFrames = numberOfFrames * this.STRORDER; + out.writeInt(numberOfSTRFrames); // number of MGC frames + for (int j = 0; j < numberOfSTRFrames; j++) { + out.writeFloat(strData.readFloat()); + } + } + } + + /** + * Initialize this component + * + * @throws MaryConfigurationException + * if there is problem with basename list + */ + @Override + protected void initialiseComp() throws Exception { + + String timelineDir = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "files"; + createDirectoryifNotExists(timelineDir); + + try { + String basenameFile = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "basenames.lst"; + if ((new File(basenameFile)).exists()) { + System.out.println("Loading basenames of vocalizations from '" + basenameFile + "' list..."); + bnlVocalizations = new BasenameList(basenameFile); + System.out.println("Found " + bnlVocalizations.getLength() + " vocalizations in basename list"); + } else { + String vocalWavDir = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "wav"; + System.out.println("Loading basenames of vocalizations from '" + vocalWavDir + "' directory..."); + bnlVocalizations = new BasenameList(vocalWavDir, ".wav"); + System.out.println("Found " + bnlVocalizations.getLength() + " vocalizations in " + vocalWavDir + " directory"); + } + } catch (IOException e) { + throw new MaryConfigurationException("Problem with basename list " + e); + } + } + + /** + * Write the header of this feature file to the given DataOutput + * + * @param out + * DataOutput to write header + * @throws IOException + * if it can't write data into DataOutput + */ + protected void writeHeaderTo(DataOutput out) throws IOException { + new MaryHeader(MaryHeader.LISTENERFEATS).writeTo(out); + } + + /** + * Create new directory if the directory doesn't exist + * + * @param dirName + * directory path + * @throws Exception + * if it fails + */ + private void createDirectoryifNotExists(String dirName) throws Exception { + if (!(new File(dirName)).exists()) { + System.out.println(dirName + " directory does not exist; "); + if (!(new File(dirName)).mkdirs()) { + throw new Exception("Could not create directory " + dirName); + } + System.out.println("Created successfully.\n"); + } + } + + @Override + public String getName() { + return "MLSAFeatureFileWriter"; + } + + @Override + public int getProgress() { + return this.progress; + } + + /** + * @param args + */ + public static void main(String[] args) { + // TODO Auto-generated method stub + + } +} diff --git a/marytts-purgatory/marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/SnackF0ContourExtractor.java b/marytts-purgatory/marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/SnackF0ContourExtractor.java new file mode 100644 index 0000000000..41152e24be --- /dev/null +++ b/marytts-purgatory/marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/SnackF0ContourExtractor.java @@ -0,0 +1,239 @@ +/** + * Copyright 2010 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ + +package marytts.tools.voiceimport.vocalizations; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.PrintWriter; +import java.util.SortedMap; +import java.util.TreeMap; + +import marytts.signalproc.analysis.SPTKPitchReaderWriter; +import marytts.tools.voiceimport.DatabaseLayout; +import marytts.tools.voiceimport.VoiceImportComponent; +import marytts.util.MaryUtils; +import marytts.util.data.text.SnackTextfileDoubleDataSource; +import marytts.util.io.BasenameList; +import marytts.util.io.General; + +public class SnackF0ContourExtractor extends VoiceImportComponent { + + String vocalizationsDir; + BasenameList bnlVocalizations; + + protected DatabaseLayout db = null; + // protected String correctedPmExt = ".pm.corrected"; + protected String snackPmExt = ".snack"; + protected String lf0Ext = ".lf0"; + protected String scriptFileName; + + private int percent = 0; + + public final String MINPITCH = "SnackF0ContourExtractor.minPitch"; + public final String MAXPITCH = "SnackF0ContourExtractor.maxPitch"; + public final String COMMAND = "SnackF0ContourExtractor.command"; + public final String LF0DIR = "SnackF0ContourExtractor.pitchContoursDir"; + public final String WAVEDIR = "SnackF0ContourExtractor.vocalizationWaveDir"; + public final String SAMPLERATE = "SnackF0ContourExtractor.samplingRate"; + public final String FRAMEPERIOD = "SnackF0ContourExtractor.framePeriod"; + + public String getName() { + return "SnackF0ContourExtractor"; + } + + @Override + protected void initialiseComp() { + scriptFileName = db.getProp(db.VOCALIZATIONSDIR) + "script.snack"; + if (!(new File(getProp(LF0DIR))).exists()) { + + System.out.println("vocalizations/lf0 directory does not exist; "); + if (!(new File(getProp(LF0DIR))).mkdirs()) { + throw new Error("Could not create vocalizations/lf0"); + } + System.out.println("Created successfully.\n"); + + } + + try { + String basenameFile = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "basenames.lst"; + if ((new File(basenameFile)).exists()) { + System.out.println("Loading basenames of vocalizations from '" + basenameFile + "' list..."); + bnlVocalizations = new BasenameList(basenameFile); + System.out.println("Found " + bnlVocalizations.getLength() + " vocalizations in basename list"); + } else { + String vocalWavDir = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "wav"; + System.out.println("Loading basenames of vocalizations from '" + vocalWavDir + "' directory..."); + bnlVocalizations = new BasenameList(vocalWavDir, ".wav"); + System.out.println("Found " + bnlVocalizations.getLength() + " vocalizations in " + vocalWavDir + " directory"); + } + } catch (IOException e) { + e.printStackTrace(); + } + } + + public SortedMap getDefaultProps(DatabaseLayout db) { + this.db = db; + if (props == null) { + props = new TreeMap(); + props.put(COMMAND, "praat"); + if (db.getProp(db.GENDER).equals("female")) { + props.put(MINPITCH, "100"); + props.put(MAXPITCH, "500"); + } else { + props.put(MINPITCH, "75"); + props.put(MAXPITCH, "300"); + } + props.put(WAVEDIR, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "wav"); + props.put(LF0DIR, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "lf0"); + props.put(FRAMEPERIOD, "80"); + props.put(SAMPLERATE, "16000"); + // vocalizationsDir = db.getProp(db.ROOTDIR)+File.separator+"vocalizations"; + if (MaryUtils.isWindows()) + props.put(COMMAND, "c:/tcl/tclsh.exe"); // TODO someone with windows, please confirm or correct + else + props.put(COMMAND, "/usr/bin/tclsh"); + } + return props; + } + + /** + * The standard compute() method of the VoiceImportComponent interface. + * + * @throws InterruptedException + */ + public boolean compute() throws IOException, InterruptedException { + + String[] baseNameArray = bnlVocalizations.getListAsArray(); + System.out.println("Computing pitchmarks for " + baseNameArray.length + " utterances."); + + File script = new File(scriptFileName); + + // What is the purpose of trunk/marytts/tools/voiceimport/pm.tcl, if it's hardcoded below here? + if (script.exists()) + script.delete(); + PrintWriter toScript = new PrintWriter(new FileWriter(script)); + toScript.println("#!" + getProp(COMMAND)); + toScript.println(" "); + toScript.println("package require snack"); + toScript.println(" "); + toScript.println("snack::sound s"); + toScript.println(" "); + toScript.println("s read [lindex $argv 0]"); + toScript.println(" "); + toScript.println("set frameperiod [ lindex $argv 4 ]"); + toScript.println("set samplerate [ lindex $argv 5 ]"); + toScript.println("set framelength [expr {double($frameperiod) / $samplerate}]"); + toScript.println(" "); + toScript.println("set fd [open [lindex $argv 1] w]"); + toScript.println("puts $fd [join [s pitch -method esps -maxpitch [lindex $argv 2] -minpitch [lindex $argv 3] -framelength $framelength] \\n]"); + // toScript.println( "puts stdout [ lindex $argv 4 ]"); + // toScript.println( "puts stdout $framelength"); + toScript.println("close $fd"); + toScript.println(" "); + toScript.println("exit"); + toScript.println(" "); + toScript.close(); + + System.out.println("Computing pitchmarks for " + baseNameArray.length + " utterances."); + + /* Ensure the existence of the target pitchmark directory */ + /* + * File dir = new File(db.getProp(PMDIR)); if (!dir.exists()) { System.out.println( "Creating the directory [" + + * db.getProp(PMDIR) + "]." ); dir.mkdir(); } + */ + + /* execute snack */ + for (int i = 0; i < baseNameArray.length; i++) { + percent = 100 * i / baseNameArray.length; + String wavFile = getProp(WAVEDIR) + baseNameArray[i] + db.getProp(db.WAVEXT); + String snackFile = getProp(LF0DIR) + baseNameArray[i] + snackPmExt; + String outLF0File = getProp(LF0DIR) + baseNameArray[i] + lf0Ext; + System.out.println("Writing pm file to " + snackFile); + + boolean isWindows = true; // TODO This is WRONG, and never used. Consider removal. + String strTmp = scriptFileName + " " + wavFile + " " + snackFile + " " + getProp(MAXPITCH) + " " + getProp(MINPITCH) + + " " + getProp(FRAMEPERIOD) + " " + getProp(SAMPLERATE); + + if (MaryUtils.isWindows()) + strTmp = "cmd.exe /c " + strTmp; + else + strTmp = getProp(COMMAND) + " " + strTmp; + + General.launchProc(strTmp, "SnackF0ContourExtractor", baseNameArray[i]); + + // Now convert the snack format into EST pm format + double[] pm = new SnackTextfileDoubleDataSource(new FileReader(snackFile)).getAllData(); + SPTKPitchReaderWriter sptkPitch = new SPTKPitchReaderWriter(pm, null); + sptkPitch.writeIntoSPTKLF0File(outLF0File); + } + return true; + + } + + /** + * @param args + */ + public static void main(String[] args) { + // TODO Auto-generated method stub + + } + + public int getProgress() { + return percent; + } + + protected void setupHelp() { + if (props2Help == null) { + props2Help = new TreeMap(); + props2Help.put(COMMAND, "The command that is used to launch snack"); + props2Help.put(MINPITCH, "minimum value for the pitch (in Hz). Default: female 100, male 75"); + props2Help.put(MAXPITCH, "maximum value for the pitch (in Hz). Default: female 500, male 300"); + } + } + + private static class StreamGobbler extends Thread { + InputStream is; + String type; + + public StreamGobbler(InputStream is, String type) { + this.is = is; + this.type = type; + } + + public void run() { + try { + InputStreamReader isr = new InputStreamReader(is); + BufferedReader br = new BufferedReader(isr); + String line = null; + while ((line = br.readLine()) != null) + System.out.println(type + ">" + line); + } catch (IOException ioe) { + ioe.printStackTrace(); + } + } + } + +} diff --git a/marytts-purgatory/marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/VocalizationF0PolyFeatureFileWriter.java b/marytts-purgatory/marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/VocalizationF0PolyFeatureFileWriter.java new file mode 100644 index 0000000000..33685a5492 --- /dev/null +++ b/marytts-purgatory/marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/VocalizationF0PolyFeatureFileWriter.java @@ -0,0 +1,484 @@ +/** + * Copyright 2006 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.tools.voiceimport.vocalizations; + +import java.awt.Color; +import java.io.BufferedOutputStream; +import java.io.BufferedReader; +import java.io.DataOutput; +import java.io.DataOutputStream; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringReader; +import java.io.StringWriter; +import java.io.UnsupportedEncodingException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.SortedMap; +import java.util.TreeMap; + +import javax.sound.sampled.AudioFormat; +import javax.swing.JFrame; + +import marytts.exceptions.MaryConfigurationException; +import marytts.features.FeatureDefinition; +import marytts.features.FeatureVector; +import marytts.signalproc.analysis.F0TrackerAutocorrelationHeuristic; +import marytts.signalproc.analysis.PitchFileHeader; +import marytts.signalproc.display.FunctionGraph; +import marytts.tools.voiceimport.DatabaseLayout; +import marytts.tools.voiceimport.VoiceImportComponent; +import marytts.unitselection.data.FeatureFileReader; +import marytts.unitselection.data.TimelineReader; +import marytts.util.data.BufferedDoubleDataSource; +import marytts.util.data.Datagram; +import marytts.util.data.DatagramDoubleDataSource; +import marytts.util.data.MaryHeader; +import marytts.util.data.audio.AudioPlayer; +import marytts.util.data.audio.DDSAudioInputStream; +import marytts.util.math.ArrayUtils; +import marytts.util.math.Polynomial; +import marytts.util.signal.SignalProcUtils; +import marytts.vocalizations.VocalizationUnitFileReader; + +/** + * NOT COMPLETED (USEFUL FOR FUTURE) + * + * @author sathish + * + */ +public class VocalizationF0PolyFeatureFileWriter extends VoiceImportComponent { + protected File maryDir; + protected FeatureFileReader features; + protected FeatureDefinition inFeatureDefinition; + protected File outFeatureFile; + protected FeatureDefinition outFeatureDefinition; + protected VocalizationUnitFileReader listenerUnits; + protected TimelineReader audio; + protected DatabaseLayout db = null; + protected int percent = 0; + + private final String name = "F0PolynomialFeatureFileWriter"; + public final String UNITFILE = name + ".unitFile"; + public final String WAVETIMELINE = name + ".waveTimeLine"; + public final String FEATUREFILE = name + ".featureFile"; + public final String F0FEATUREFILE = name + ".f0FeatureFile"; + public final String POLYNOMORDER = name + ".polynomOrder"; + public final String SHOWGRAPH = name + ".showGraph"; + public final String INTERPOLATE = name + ".interpolate"; + public final String MINPITCH = name + ".minPitch"; + public final String MAXPITCH = name + ".maxPitch"; + + public String getName() { + return name; + } + + public SortedMap getDefaultProps(DatabaseLayout db) { + this.db = db; + if (props == null) { + props = new TreeMap(); + String fileDir = db.getProp(db.FILEDIR); + String maryExt = db.getProp(db.MARYEXT); + props.put(UNITFILE, fileDir + "halfphoneUnits" + maryExt); + props.put(WAVETIMELINE, fileDir + "timeline_waveforms" + maryExt); + props.put(FEATUREFILE, fileDir + "halfphoneFeatures" + maryExt); + props.put(F0FEATUREFILE, fileDir + "vocalizationF0Polynomials" + maryExt); + props.put(POLYNOMORDER, "3"); + props.put(SHOWGRAPH, "false"); + props.put(INTERPOLATE, "true"); + if (db.getProp(db.GENDER).equals("female")) { + props.put(MINPITCH, "100"); + props.put(MAXPITCH, "600"); + } else { + props.put(MINPITCH, "60"); + props.put(MAXPITCH, "400"); + } + } + return props; + } + + protected void setupHelp() { + if (props2Help == null) { + props2Help = new TreeMap(); + props2Help.put(UNITFILE, "file containing all halfphone units"); + props2Help.put(WAVETIMELINE, "file containing all waveforms or models that can genarate them"); + props2Help.put(FEATUREFILE, "file containing all halfphone units and their target cost features"); + props2Help.put(F0FEATUREFILE, "file containing syllable-based polynom coefficients on vowels"); + props2Help.put(POLYNOMORDER, "order of the polynoms used to approximate syllable F0 curves"); + props2Help.put(SHOWGRAPH, "whether to show a graph with f0 aproximations for each sentence"); + props2Help.put(INTERPOLATE, "whether to interpolate F0 across unvoiced regions"); + props2Help.put(MINPITCH, "minimum value for the pitch (in Hz). Default: female 100, male 75"); + props2Help.put(MAXPITCH, "maximum value for the pitch (in Hz). Default: female 500, male 300"); + } + } + + @Override + public boolean compute() throws IOException, MaryConfigurationException { + logger.info("F0 polynomial feature file writer started."); + + maryDir = new File(db.getProp(db.FILEDIR)); + if (!maryDir.exists()) { + maryDir.mkdirs(); + System.out.println("Created the output directory [" + (db.getProp(db.FILEDIR)) + "] to store the feature file."); + } + listenerUnits = new VocalizationUnitFileReader(getProp(UNITFILE)); + audio = new TimelineReader(getProp(WAVETIMELINE)); + + // features = new FeatureFileReader(getProp(FEATUREFILE)); + // inFeatureDefinition = features.getFeatureDefinition(); + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw); + pw.println(FeatureDefinition.BYTEFEATURES); // no byte features + pw.println(FeatureDefinition.SHORTFEATURES); // no short features + pw.println(FeatureDefinition.CONTINUOUSFEATURES); + int polynomOrder = Integer.parseInt(getProp(POLYNOMORDER)); + for (int i = polynomOrder; i >= 0; i--) { + pw.println("0 linear | f0contour_a" + i); + } + pw.close(); + String fd = sw.toString(); + logger.debug("Generated the following feature definition:"); + logger.debug(fd); + StringReader sr = new StringReader(fd); + BufferedReader br = new BufferedReader(sr); + outFeatureDefinition = new FeatureDefinition(br, true); + + outFeatureFile = new File(getProp(F0FEATUREFILE)); + DataOutputStream out = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(outFeatureFile))); + writeHeaderTo(out); + writeUnitFeaturesTo(out); + out.close(); + logger.debug("Number of processed units: " + listenerUnits.getNumberOfUnits()); + + FeatureFileReader tester = FeatureFileReader.getFeatureFileReader(getProp(F0FEATUREFILE)); + int unitsOnDisk = tester.getNumberOfUnits(); + if (unitsOnDisk == listenerUnits.getNumberOfUnits()) { + System.out.println("Can read right number of units"); + return true; + } else { + System.out.println("Read wrong number of units: " + unitsOnDisk); + return false; + } + } + + /** + * @param out + * @throws IOException + * @throws UnsupportedEncodingException + * @throws FileNotFoundException + */ + protected void writeUnitFeaturesTo(DataOutput out) throws IOException, UnsupportedEncodingException, FileNotFoundException { + int numUnits = listenerUnits.getNumberOfUnits(); + int unitSampleRate = listenerUnits.getSampleRate(); + int audioSampleRate = audio.getSampleRate(); + boolean showGraph = Boolean.parseBoolean(getProp(SHOWGRAPH)); + boolean interpolate = Boolean.parseBoolean(getProp(INTERPOLATE)); + int polynomOrder = Integer.parseInt(getProp(POLYNOMORDER)); + float[] zeros = new float[polynomOrder + 1]; + int unitIndex = 0; + + out.writeInt(numUnits); + logger.debug("Number of units : " + numUnits); + + FeatureDefinition featureDefinition = features.getFeatureDefinition(); + int fiPhoneme = featureDefinition.getFeatureIndex("phone"); + byte fvPhoneme_0 = featureDefinition.getFeatureValueAsByte(fiPhoneme, "0"); + byte fvPhoneme_Silence = featureDefinition.getFeatureValueAsByte(fiPhoneme, "_"); + int fiLR = featureDefinition.getFeatureIndex("halfphone_lr"); + byte fvLR_L = featureDefinition.getFeatureValueAsByte(fiLR, "L"); + byte fvLR_R = featureDefinition.getFeatureValueAsByte(fiLR, "R"); + int fiSylStart = featureDefinition.getFeatureIndex("segs_from_syl_start"); + int fiSylEnd = featureDefinition.getFeatureIndex("segs_from_syl_end"); + int fiSentenceStart = featureDefinition.getFeatureIndex("words_from_sentence_start"); + int fiSentenceEnd = featureDefinition.getFeatureIndex("words_from_sentence_end"); + int fiWordStart = featureDefinition.getFeatureIndex("segs_from_word_start"); + int fiWordEnd = featureDefinition.getFeatureIndex("segs_from_word_end"); + int fiVowel = featureDefinition.getFeatureIndex("ph_vc"); + byte fvVowel_Plus = featureDefinition.getFeatureValueAsByte(fiVowel, "+"); + + boolean haveUnitLogF0 = false; + int fiUnitLogF0 = -1; + int fiUnitLogF0delta = -1; + if (featureDefinition.hasFeature("unit_logf0") && featureDefinition.hasFeature("unit_logf0delta")) { + haveUnitLogF0 = true; + fiUnitLogF0 = featureDefinition.getFeatureIndex("unit_logf0"); + fiUnitLogF0delta = featureDefinition.getFeatureIndex("unit_logf0delta"); + } + + FunctionGraph f0Graph = null; + JFrame jf = null; + int iSentenceStart = -1; + int iSentenceEnd = -1; + List iSylStarts = new ArrayList(); + List iSylEnds = new ArrayList(); + List iSylVowels = new ArrayList(); + if (showGraph) { + f0Graph = new FunctionGraph(0, 1, new double[1]); + f0Graph.setYMinMax(50, 300); + f0Graph.setPrimaryDataSeriesStyle(Color.BLUE, FunctionGraph.DRAW_DOTS, FunctionGraph.DOT_FULLCIRCLE); + jf = f0Graph.showInJFrame("Sentence", false, true); + } + + for (int i = 0; i < numUnits; i++) { + percent = 100 * i / numUnits; + FeatureVector fv = features.getFeatureVector(i); + // System.out.print(featureDefinition.getFeatureValueAsString("phone", fv)); + // if (fv.getByteFeature(fiPhoneme) == fvPhoneme_0 + // || fv.getByteFeature(fiPhoneme) == fvPhoneme_Silence) continue; + if (iSentenceStart == -1 && fv.getByteFeature(fiSentenceStart) == 0 && fv.getByteFeature(fiWordStart) == 0 + && fv.getByteFeature(fiLR) == fvLR_L) { // first unit in sentence + iSentenceStart = i; + iSylStarts.clear(); + iSylEnds.clear(); + iSylVowels.clear(); + // System.out.print(", is sentence start"); + } + // Silence and edge units cannot be part of syllables, but they can + // mark start/end of sentence: + if (fv.getByteFeature(fiPhoneme) != fvPhoneme_0 && fv.getByteFeature(fiPhoneme) != fvPhoneme_Silence) { + if (fv.getByteFeature(fiSylStart) == 0 && fv.getByteFeature(fiLR) == fvLR_L) { // first segment in syllable + if (iSylStarts.size() > iSylEnds.size()) { + System.err.println("Syllable ends before other syllable starts!"); + } + iSylStarts.add(i); + // System.out.print(", is syl start"); + } + if (fv.getByteFeature(fiVowel) == fvVowel_Plus && iSylVowels.size() < iSylStarts.size()) { // first vowel unit in + // syllable + iSylVowels.add(i); + // System.out.print(", is vowel"); + } + if (fv.getByteFeature(fiSylEnd) == 0 && fv.getByteFeature(fiLR) == fvLR_R) { // last segment in syllable + iSylEnds.add(i); + // System.out.print(", is syl end"); + assert iSylStarts.size() == iSylEnds.size(); + if (iSylVowels.size() < iSylEnds.size()) { + // System.err.println("Syllable contains no vowel -- skipping"); + iSylStarts.remove(iSylStarts.size() - 1); + iSylEnds.remove(iSylEnds.size() - 1); + } + } + } + if (iSentenceStart != -1 && fv.getByteFeature(fiSentenceEnd) == 0 && fv.getByteFeature(fiWordEnd) == 0 + && fv.getByteFeature(fiLR) == fvLR_R) { // last unit in sentence + iSentenceEnd = i; + // System.out.print(", is sentence end"); + if (iSylEnds.size() < iSylStarts.size()) { + System.err.println("Last syllable in sentence is not properly closed"); + iSylEnds.add(i); + } + } + // System.out.println(); + + if (iSentenceStart >= 0 && iSentenceEnd >= iSentenceStart && iSylVowels.size() > 0) { + assert iSylStarts.size() == iSylEnds.size() : "Have " + iSylStarts.size() + " syllable starts, but " + + iSylEnds.size() + " syllable ends!"; + assert iSylStarts.size() == iSylVowels.size(); + long tsSentenceStart = listenerUnits.getUnit(iSentenceStart).startTime; + long tsSentenceEnd = listenerUnits.getUnit(iSentenceEnd).startTime + listenerUnits.getUnit(iSentenceEnd).duration; + long tsSentenceDuration = tsSentenceEnd - tsSentenceStart; + Datagram[] sentenceData = audio.getDatagrams(tsSentenceStart, tsSentenceDuration); + DatagramDoubleDataSource ddds = new DatagramDoubleDataSource(sentenceData); + double[] sentenceAudio = ddds.getAllData(); + AudioPlayer ap = null; + if (showGraph) { + ap = new AudioPlayer(new DDSAudioInputStream(new BufferedDoubleDataSource(sentenceAudio), new AudioFormat( + AudioFormat.Encoding.PCM_SIGNED, audioSampleRate, // samples per second + 16, // bits per sample + 1, // mono + 2, // nr. of bytes per frame + audioSampleRate, // nr. of frames per second + true))); // big-endian;)) + ap.start(); + } + PitchFileHeader params = new PitchFileHeader(); + params.fs = audioSampleRate; + params.minimumF0 = Double.parseDouble(getProp(MINPITCH)); + params.maximumF0 = Double.parseDouble(getProp(MAXPITCH)); + F0TrackerAutocorrelationHeuristic tracker = new F0TrackerAutocorrelationHeuristic(params); + tracker.pitchAnalyze(new BufferedDoubleDataSource(sentenceAudio)); + double frameShiftTime = tracker.getSkipSizeInSeconds(); + double[] f0Array = tracker.getF0Contour(); + if (f0Array != null) { + for (int j = 0; j < f0Array.length; j++) { + if (f0Array[j] == 0) { + f0Array[j] = Double.NaN; + } + } + if (f0Array.length >= 3) { + f0Array = SignalProcUtils.medianFilter(f0Array, 5); + } + if (showGraph) { + f0Graph.updateData(0, tsSentenceDuration / (double) audioSampleRate / f0Array.length, f0Array); + jf.repaint(); + } + + double[] f0AndInterpol; + if (interpolate) { + double[] interpol = new double[f0Array.length]; + Arrays.fill(interpol, Double.NaN); + f0AndInterpol = new double[f0Array.length]; + int iLastValid = -1; + for (int j = 0; j < f0Array.length; j++) { + if (!Double.isNaN(f0Array[j])) { // a valid value + if (iLastValid == j - 1) { + // no need to interpolate + f0AndInterpol[j] = f0Array[j]; + } else { + // need to interpolate + double prevF0; + if (iLastValid < 0) { // we don't have a previous value -- use current one + prevF0 = f0Array[j]; + } else { + prevF0 = f0Array[iLastValid]; + } + double delta = (f0Array[j] - prevF0) / (j - iLastValid); + double f0 = prevF0; + for (int k = iLastValid + 1; k < j; k++) { + f0 += delta; + interpol[k] = f0; + f0AndInterpol[k] = f0; + } + } + iLastValid = j; + } + } + if (showGraph) { + f0Graph.addDataSeries(interpol, Color.GREEN, FunctionGraph.DRAW_DOTS, FunctionGraph.DOT_EMPTYCIRCLE); + jf.repaint(); + } + } else { + f0AndInterpol = f0Array.clone(); + } + + for (int j = 0; j < f0AndInterpol.length; j++) { + if (f0AndInterpol[j] == 0) + f0AndInterpol[j] = Double.NaN; + else + f0AndInterpol[j] = Math.log(f0AndInterpol[j]); + } + double[] approx = new double[f0Array.length]; + Arrays.fill(approx, Double.NaN); + for (int s = 0; s < iSylStarts.size(); s++) { + long tsSylStart = listenerUnits.getUnit(iSylStarts.get(s)).startTime; + long tsSylEnd = listenerUnits.getUnit(iSylEnds.get(s)).startTime + + listenerUnits.getUnit(iSylEnds.get(s)).duration; + long tsSylDuration = tsSylEnd - tsSylStart; + int iSylVowel = iSylVowels.get(s); + // now map time to position in f0AndInterpol array: + int iSylStart = (int) (((double) (tsSylStart - tsSentenceStart) / tsSentenceDuration) * f0AndInterpol.length); + assert iSylStart >= 0; + int iSylEnd = iSylStart + (int) ((double) tsSylDuration / tsSentenceDuration * f0AndInterpol.length) + 1; + if (iSylEnd > f0AndInterpol.length) + iSylEnd = f0AndInterpol.length; + // System.out.println("Syl "+s+" from "+iSylStart+" to "+iSylEnd+" out of "+f0AndInterpol.length); + double[] sylF0 = new double[iSylEnd - iSylStart]; + System.arraycopy(f0AndInterpol, iSylStart, sylF0, 0, sylF0.length); + double[] coeffs = Polynomial.fitPolynomial(sylF0, polynomOrder); + if (coeffs != null) { + if (showGraph) { + double[] sylPred = Polynomial.generatePolynomialValues(coeffs, sylF0.length, 0, 1); + System.arraycopy(sylPred, 0, approx, iSylStart, sylPred.length); + } + // Write coefficients to file + while (unitIndex < iSylVowel) { + FeatureVector outFV = outFeatureDefinition.toFeatureVector(unitIndex, null, null, zeros); + outFV.writeTo(out); + unitIndex++; + } + float[] fcoeffs = ArrayUtils.copyDouble2Float(coeffs); + // System.out.print("Polynomial values (unit "+unitIndex+") "); + // for (int p=0; p. + * + */ +package marytts.tools.voiceimport.vocalizations; + +import java.awt.Color; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileWriter; +import java.io.IOException; +import java.io.PrintWriter; +import java.io.UnsupportedEncodingException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Set; +import java.util.SortedMap; +import java.util.TreeMap; + +import javax.sound.sampled.AudioFormat; +import javax.sound.sampled.AudioInputStream; +import javax.sound.sampled.AudioSystem; +import javax.sound.sampled.UnsupportedAudioFileException; +import javax.swing.JFrame; + +import marytts.features.FeatureDefinition; +import marytts.signalproc.analysis.F0TrackerAutocorrelationHeuristic; +import marytts.signalproc.analysis.PitchFileHeader; +import marytts.signalproc.analysis.PitchReaderWriter; +import marytts.signalproc.analysis.SPTKPitchReaderWriter; +import marytts.signalproc.analysis.distance.DistanceComputer; +import marytts.signalproc.display.FunctionGraph; +import marytts.tools.voiceimport.DatabaseLayout; +import marytts.tools.voiceimport.VoiceImportComponent; +import marytts.unitselection.data.FeatureFileReader; +import marytts.unitselection.data.TimelineReader; +import marytts.unitselection.data.UnitFileReader; +import marytts.util.data.BufferedDoubleDataSource; +import marytts.util.data.audio.AudioDoubleDataSource; +import marytts.util.data.audio.AudioPlayer; +import marytts.util.data.audio.DDSAudioInputStream; +import marytts.util.io.BasenameList; +import marytts.util.math.Polynomial; +import marytts.util.signal.SignalProcUtils; +import marytts.vocalizations.KMeansClusterer; + +public class VocalizationF0PolynomialInspector extends VoiceImportComponent { + protected FeatureFileReader features; + protected FeatureDefinition inFeatureDefinition; + protected UnitFileReader units; + protected FeatureFileReader contours; + protected TimelineReader audio; + protected DatabaseLayout db = null; + protected int percent = 0; + protected FunctionGraph f0Graph = null; + protected JFrame jf = null; + protected PrintWriter featurePW; + protected double costMeasure = 0; + private HashMap minF0Values; + private HashMap maxF0Values; + private Set characters; + + protected BasenameList bnlVocalizations; + + private final String name = "VocalizationF0PolynomialInspector"; + public final String WAVEDIR = name + ".waveDir"; + public final String F0POLYFILE = name + ".f0PolynomialFeatureFile"; + public final String F0MIN = name + ".f0Minimum"; + public final String F0MAX = name + ".f0Maximum"; + public final String PARTBASENAME = name + ".partBaseName"; + public final String ONEWORD = name + ".oneWordDescription"; + public final String KCLUSTERS = name + ".numberOfClusters"; + public final String POLYORDER = name + ".polynomialOrder"; + public final String ISEXTERNALF0 = name + ".isExternalF0Usage"; + public final String EXTERNALF0FORMAT = name + ".externalF0Format"; + public final String EXTERNALDIR = name + ".externalF0Directory"; + public final String EXTERNALEXT = name + ".externalF0Extention"; + + public String getName() { + return name; + } + + public SortedMap getDefaultProps(DatabaseLayout db) { + this.db = db; + if (props == null) { + props = new TreeMap(); + props.put(WAVEDIR, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "wav"); + props.put(F0POLYFILE, "VocalizationF0PolyFeatureFile.txt"); + props.put(PARTBASENAME, ""); + props.put(ONEWORD, ""); + props.put(F0MIN, "50"); + props.put(F0MAX, "500"); + props.put(KCLUSTERS, "15"); + props.put(POLYORDER, "3"); + props.put(ISEXTERNALF0, "true"); + props.put(EXTERNALF0FORMAT, "sptk"); + props.put(EXTERNALDIR, "lf0"); + props.put(EXTERNALEXT, ".lf0"); + } + return props; + } + + protected void setupHelp() { + if (props2Help == null) { + props2Help = new TreeMap(); + props2Help.put(WAVEDIR, "dir containing all waveforms "); + } + } + + @Override + protected void initialiseComp() { + minF0Values = new HashMap(); + maxF0Values = new HashMap(); + minF0Values.put("Spike", 50); + minF0Values.put("Poppy", 170); + minF0Values.put("Obadiah", 70); + minF0Values.put("Prudence", 130); + maxF0Values.put("Spike", 150); + maxF0Values.put("Poppy", 380); + maxF0Values.put("Obadiah", 150); + maxF0Values.put("Prudence", 280); + + characters = new HashSet(); + characters.add("Spike"); + characters.add("Poppy"); + characters.add("Obadiah"); + characters.add("Prudence"); + + try { + String basenameFile = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "basenames.lst"; + if ((new File(basenameFile)).exists()) { + System.out.println("Loading basenames of vocalizations from '" + basenameFile + "' list..."); + bnlVocalizations = new BasenameList(basenameFile); + System.out.println("Found " + bnlVocalizations.getLength() + " vocalizations in basename list"); + } else { + String vocalWavDir = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "wav"; + System.out.println("Loading basenames of vocalizations from '" + vocalWavDir + "' directory..."); + bnlVocalizations = new BasenameList(vocalWavDir, ".wav"); + System.out.println("Found " + bnlVocalizations.getLength() + " vocalizations in " + vocalWavDir + " directory"); + } + } catch (IOException e) { + e.printStackTrace(); + } + } + + public boolean compute() throws IOException, UnsupportedAudioFileException { + logger.info("F0 polynomial feature file writer started."); + f0Graph = new FunctionGraph(0, 1, new double[1]); + f0Graph.setYMinMax(50, 550); + f0Graph.setPrimaryDataSeriesStyle(Color.BLUE, FunctionGraph.DRAW_DOTS, FunctionGraph.DOT_FULLCIRCLE); + jf = f0Graph.showInJFrame("Sentence", false, true); + + String outPutFile = db.getProp(db.ROOTDIR) + File.separator + getProp(ONEWORD) + getProp(PARTBASENAME) + + getProp(F0POLYFILE); + featurePW = new PrintWriter(new FileWriter(new File(outPutFile))); + + for (int i = 0; i < bnlVocalizations.getLength(); i++) { + percent = 100 * i / bnlVocalizations.getLength(); + displaySentences(bnlVocalizations.getName(i)); + + } + featurePW.flush(); + featurePW.close(); + System.out.println("Total Cost : " + costMeasure / (double) bnlVocalizations.getLength()); + + // String fileName = "/home/sathish/phd/voices/en-GB-listener/vocal-polynomials/SpiVocalizationF0PolyFeatureFile.txt"; + int kValue = (new Integer(getProp(KCLUSTERS))).intValue(); + KMeansClusterer kmc = new KMeansClusterer(); + kmc.loadF0Polynomials(outPutFile); + kmc.trainer(kValue); + // System.exit(0); + + return true; + } + + /** + * @param out + * @throws IOException + * @throws UnsupportedAudioFileException + * @throws UnsupportedEncodingException + * @throws FileNotFoundException + */ + protected void displaySentences(String baseName) throws IOException, UnsupportedAudioFileException { + /* + * int numUnits = units.getNumberOfUnits(); int unitSampleRate = units.getSampleRate(); int audioSampleRate = + * audio.getSampleRate(); int unitIndex = 0; + * + * logger.debug("Number of units : "+numUnits); + */ + String partBaseName = getProp(PARTBASENAME).trim(); + if (!"".equals(partBaseName) && !baseName.contains(partBaseName)) { + return; + } + + /* + * String targetDescription = getProp(ONEWORD).trim(); String textFileName = + * db.getProp(db.TEXTDIR)+File.separator+baseName+db.getProp(db.TEXTEXT); String audioDescriprion = + * FileUtils.getFileAsString(new File(textFileName), "UTF-8"); + * + * if ( !"".equals(targetDescription) && !targetDescription.equals(audioDescriprion.trim())) { return; } + */ + + // if ( !baseName.equals("") && !baseName.contains(getProp(PARTBASENAME)) ){ + // return; + // } + + String waveFile = getProp(WAVEDIR) + File.separator + baseName + db.getProp(db.WAVEXT); + AudioInputStream inputAudio = AudioSystem.getAudioInputStream(new File(waveFile)); + + // Enforce PCM_SIGNED encoding + if (!inputAudio.getFormat().getEncoding().equals(AudioFormat.Encoding.PCM_SIGNED)) { + inputAudio = AudioSystem.getAudioInputStream(AudioFormat.Encoding.PCM_SIGNED, inputAudio); + } + + int audioSampleRate = (int) inputAudio.getFormat().getSampleRate(); + AudioDoubleDataSource signal = new AudioDoubleDataSource(inputAudio); + double[] sentenceAudio = signal.getAllData(); // Copies all samples in wav file into a double buffer + long tsSentenceDuration = sentenceAudio.length; + + /* + * Datagram[] sentenceData = audio.getDatagrams(tsSentenceStart, tsSentenceDuration); DatagramDoubleDataSource ddds = new + * DatagramDoubleDataSource(sentenceData); double[] sentenceAudio = ddds.getAllData(); + */ + + AudioPlayer ap = null; + ap = new AudioPlayer(new DDSAudioInputStream(new BufferedDoubleDataSource(sentenceAudio), new AudioFormat( + AudioFormat.Encoding.PCM_SIGNED, audioSampleRate, // samples per second + 16, // bits per sample + 1, // mono + 2, // nr. of bytes per frame + audioSampleRate, // nr. of frames per second + true))); // big-endian;)) + ap.start(); + + PitchFileHeader params = new PitchFileHeader(); + // params.minimumF0 = 50; + // params.maximumF0 = 250; + // params.minimumF0 = (new Double(getProp(F0MIN))).doubleValue(); + // params.maximumF0 = (new Double(getProp(F0MAX))).doubleValue(); + String character = getCharacterName(baseName); + params.minimumF0 = (minF0Values.get(character)).doubleValue(); + params.maximumF0 = (maxF0Values.get(character)).doubleValue(); + params.fs = audioSampleRate; + + double[] f0Array = null; + + if ("true".equals(getProp(ISEXTERNALF0))) { + + String externalFormat = getProp(EXTERNALF0FORMAT); + String externalDir = getProp(EXTERNALDIR); + String externalExt = getProp(EXTERNALEXT); + + if ("sptk".equals(externalFormat)) { + String fileName = db.getProp(db.VOCALIZATIONSDIR) + File.separator + externalDir + File.separator + baseName + + externalExt; + SPTKPitchReaderWriter sprw = new SPTKPitchReaderWriter(fileName); + f0Array = sprw.getF0Contour(); + } else if ("ptc".equals(externalFormat)) { + String fileName = db.getProp(db.ROOTDIR) + File.separator + externalDir + File.separator + baseName + externalExt; + PitchReaderWriter sprw = new PitchReaderWriter(fileName); + f0Array = sprw.contour; + } + } else { + F0TrackerAutocorrelationHeuristic tracker = new F0TrackerAutocorrelationHeuristic(params); + tracker.pitchAnalyze(new BufferedDoubleDataSource(sentenceAudio)); + // double frameShiftTime = tracker.getSkipSizeInSeconds(); + f0Array = tracker.getF0Contour(); + } + + // f0Array = cutStartEndUnvoicedSegments(f0Array); + + if (f0Array != null) { + for (int j = 0; j < f0Array.length; j++) { + if (f0Array[j] == 0) { + f0Array[j] = Double.NaN; + } + } + if (f0Array.length >= 3) { + f0Array = SignalProcUtils.medianFilter(f0Array, 5); + } + f0Graph.updateData(0, tsSentenceDuration / (double) audioSampleRate / f0Array.length, f0Array); + jf.repaint(); + + double[] f0AndInterpol; + double[] interpol = new double[f0Array.length]; + Arrays.fill(interpol, Double.NaN); + f0AndInterpol = new double[f0Array.length]; + int iLastValid = -1; + for (int j = 0; j < f0Array.length; j++) { + if (!Double.isNaN(f0Array[j])) { // a valid value + if (iLastValid == j - 1) { + // no need to interpolate + f0AndInterpol[j] = f0Array[j]; + } else { + // need to interpolate + double prevF0; + if (iLastValid < 0) { // we don't have a previous value -- use current one + prevF0 = f0Array[j]; + } else { + prevF0 = f0Array[iLastValid]; + } + double delta = (f0Array[j] - prevF0) / (j - iLastValid); + double f0 = prevF0; + for (int k = iLastValid + 1; k < j; k++) { + f0 += delta; + interpol[k] = f0; + f0AndInterpol[k] = f0; + } + } + iLastValid = j; + } + } + f0Graph.addDataSeries(interpol, Color.GREEN, FunctionGraph.DRAW_DOTS, FunctionGraph.DOT_EMPTYCIRCLE); + jf.repaint(); + + double[] f0AndInterpolate = combineF0andInterpolate(f0Array, interpol); + + // double[] coeffs = Polynomial.fitPolynomial(sylF0, polynomOrder); + int polynomialOrder = (new Integer(getProp(POLYORDER))).intValue(); + double[] coeffs = Polynomial.fitPolynomial(f0AndInterpolate, polynomialOrder); + + if (coeffs != null) { + double[] sylPred = Polynomial.generatePolynomialValues(coeffs, interpol.length, 0, 1); + f0Graph.addDataSeries(sylPred, Color.RED, FunctionGraph.DRAW_LINE, -1); + double eqDistance = DistanceComputer.getEuclideanDistance(sylPred, f0AndInterpolate); + System.out.println(baseName + " - EqDist: " + eqDistance / (double) sylPred.length); + costMeasure += (eqDistance / (double) sylPred.length); + featurePW.print(baseName + " "); + for (double c : coeffs) { + featurePW.print(c + " "); + } + featurePW.println(); + + } + + // double[] pred = Polynomial.generatePolynomialValues(coeffs, iUnitDurationInArray, 0, 1); + // f0Graph.addDataSeries(unitF0, Color.BLACK, FunctionGraph.DRAW_LINE, -1); + + try { + ap.join(); + Thread.sleep(10); + } catch (InterruptedException ie) { + } + // System.out.println(); + + } + + } + + private double[] cutStartEndUnvoicedSegments(double[] array) { + + if (array == null) + return null; + + int startIndex = 0; + int endIndex = array.length; + + // find start index + for (int i = 0; i < array.length; i++) { + if (array[i] != 0) { + startIndex = i; + break; + } + } + + // find end index + for (int i = (array.length - 1); i > 0; i--) { + if (array[i] != 0) { + endIndex = i; + break; + } + } + + int newArraySize = endIndex - startIndex; + + double[] newArray = new double[newArraySize]; + System.arraycopy(array, startIndex, newArray, 0, newArraySize); + + for (int i = 0; i < newArray.length; i++) { + System.out.println(newArray[i]); + } + System.out.println("Resized from " + array.length + " to " + newArraySize); + + return newArray; + } + + private String getCharacterName(String baseName) { + Iterator it = characters.iterator(); + while (it.hasNext()) { + String character = it.next().trim(); + if (baseName.startsWith(character)) { + return character; + } + } + return null; + } + + private double[] combineF0andInterpolate(double[] f0Array, double[] interpol) { + + double[] f0AndInterpolate = new double[f0Array.length]; + Arrays.fill(f0AndInterpolate, Double.NaN); + for (int i = 0; i < f0Array.length; i++) { + if (!Double.isNaN(f0Array[i])) { + f0AndInterpolate[i] = f0Array[i]; + } else if (!Double.isNaN(interpol[i])) { + f0AndInterpolate[i] = interpol[i]; + } + // System.out.println(f0Array[i]+" "+interpol[i]+" "+f0AndInterpolate[i]); + } + + return f0AndInterpolate; + } + + /** + * Provide the progress of computation, in percent, or -1 if that feature is not implemented. + * + * @return -1 if not implemented, or an integer between 0 and 100. + */ + public int getProgress() { + return percent; + } + + /** + * @param args + */ + public static void main(String[] args) throws Exception { + VocalizationF0PolynomialInspector acfeatsWriter = new VocalizationF0PolynomialInspector(); + DatabaseLayout db = new DatabaseLayout(acfeatsWriter); + acfeatsWriter.compute(); + } + +} diff --git a/marytts-purgatory/marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/VocalizationFeatureFileWriter.java b/marytts-purgatory/marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/VocalizationFeatureFileWriter.java new file mode 100644 index 0000000000..f1d4558671 --- /dev/null +++ b/marytts-purgatory/marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/VocalizationFeatureFileWriter.java @@ -0,0 +1,278 @@ +/** + * Copyright 2006 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.tools.voiceimport.vocalizations; + +import java.io.BufferedOutputStream; +import java.io.BufferedReader; +import java.io.DataOutput; +import java.io.DataOutputStream; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.FileReader; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; +import java.util.SortedMap; +import java.util.TreeMap; + +import marytts.exceptions.MaryConfigurationException; +import marytts.features.FeatureDefinition; +import marytts.features.FeatureVector; +import marytts.tools.voiceimport.DatabaseLayout; +import marytts.tools.voiceimport.VoiceImportComponent; +import marytts.util.data.MaryHeader; +import marytts.util.io.BasenameList; +import marytts.vocalizations.VocalizationAnnotationReader; +import marytts.vocalizations.VocalizationFeatureFileReader; +import marytts.vocalizations.VocalizationUnitFileReader; + +/** + * vocalization feature file writer + * + * @author sathish + * + */ +public class VocalizationFeatureFileWriter extends VoiceImportComponent { + + protected File outFeatureFile; + protected FeatureDefinition featureDefinition; + protected VocalizationUnitFileReader listenerUnits; + + protected DatabaseLayout db = null; + protected int percent = 0; + private ArrayList featureCategories; // feature categories + private Map> annotationData; // basename --> (feature category, feature value) + + protected String vocalizationsDir; + protected BasenameList bnlVocalizations; + + private final String name = "VocalizationFeatureFileWriter"; + public final String UNITFILE = name + ".unitFile"; + public final String FEATUREFILE = name + ".featureFile"; + public final String MANUALFEATURES = name + ".annotationFeatureFile"; + public final String FEATDEF = name + ".featureDefinition"; + + public final String POLYNOMORDER = name + ".polynomOrder"; + public final String SHOWGRAPH = name + ".showGraph"; + public final String INTERPOLATE = name + ".interpolate"; + public final String MINPITCH = name + ".minPitch"; + public final String MAXPITCH = name + ".maxPitch"; + + // public String BASELIST = name + ".backchannelBNL"; + + public String getName() { + return name; + } + + public SortedMap getDefaultProps(DatabaseLayout db) { + this.db = db; + if (props == null) { + props = new TreeMap(); + String fileDir = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "files" + File.separator; + String maryExt = db.getProp(db.MARYEXT); + props.put(UNITFILE, fileDir + "vocalization_units" + maryExt); + props.put(FEATUREFILE, fileDir + "vocalization_features" + maryExt); + props.put(MANUALFEATURES, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "features" + File.separator + + "annotation_vocalizations_features.txt"); + props.put(FEATDEF, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "features" + File.separator + + "vocalization_feature_definition.txt"); + } + return props; + } + + protected void setupHelp() { + if (props2Help == null) { + props2Help = new TreeMap(); + props2Help.put(UNITFILE, "file containing all halfphone units"); + props2Help.put(FEATUREFILE, "file containing all halfphone units and their target cost features"); + } + } + + @Override + protected void initialiseComp() { + + featureCategories = new ArrayList(); // feature categories + annotationData = new HashMap>(); // basename --> (feature category, feature value) + + try { + String basenameFile = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "basenames.lst"; + if ((new File(basenameFile)).exists()) { + System.out.println("Loading basenames of vocalizations from '" + basenameFile + "' list..."); + bnlVocalizations = new BasenameList(basenameFile); + System.out.println("Found " + bnlVocalizations.getLength() + " vocalizations in basename list"); + } else { + String vocalWavDir = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "wav"; + System.out.println("Loading basenames of vocalizations from '" + vocalWavDir + "' directory..."); + bnlVocalizations = new BasenameList(vocalWavDir, ".wav"); + System.out.println("Found " + bnlVocalizations.getLength() + " vocalizations in " + vocalWavDir + " directory"); + } + } catch (IOException e) { + e.printStackTrace(); + } + } + + @Override + public boolean compute() throws IOException, MaryConfigurationException { + // read feature definition + BufferedReader fDBfr = new BufferedReader(new FileReader(new File(getProp(FEATDEF)))); + // featureDefinition = new FeatureDefinition(fDBfr, true); + featureDefinition = new FeatureDefinition(fDBfr, true); + listenerUnits = new VocalizationUnitFileReader(getProp(UNITFILE)); + + // load annotation + VocalizationAnnotationReader annotationReader = new VocalizationAnnotationReader(getProp(MANUALFEATURES), + bnlVocalizations); + annotationData = annotationReader.getVocalizationsAnnotation(); + featureCategories = annotationReader.getFeatureList(); + + // write features into timeline file + DataOutputStream out = new DataOutputStream( + new BufferedOutputStream(new FileOutputStream(new File(getProp(FEATUREFILE))))); + writeHeaderTo(out); + writeUnitFeaturesTo(out); + out.close(); + logger.debug("Number of processed units: " + listenerUnits.getNumberOfUnits()); + + VocalizationFeatureFileReader tester = new VocalizationFeatureFileReader(getProp(FEATUREFILE)); + int unitsOnDisk = tester.getNumberOfUnits(); + if (unitsOnDisk == listenerUnits.getNumberOfUnits()) { + System.out.println("Can read right number of units"); + return true; + } else { + System.out.println("Read wrong number of units: " + unitsOnDisk); + return false; + } + } + + protected void writeUnitFeaturesTo(DataOutput out) throws IOException, UnsupportedEncodingException, FileNotFoundException { + + int numUnits = listenerUnits.getNumberOfUnits(); + + out.writeInt(numUnits); + // logger.debug("Number of vocalization units : "+numUnits); + System.out.println("Number of vocalization units : " + numUnits); + System.out.println("Annotation number of units: " + annotationData.size()); + if (annotationData.size() != listenerUnits.getNumberOfUnits()) { + throw new RuntimeException("Number of units in vocalizations unit-file is not equal to number of basenames. "); + } + + if (featureCategories.size() != featureDefinition.getNumberOfFeatures()) { + throw new RuntimeException( + "Number of categories in feature_definition is not equal to features given in annotation file "); + } + + /** + * TODO sanity check for each basename + */ + + int noOfFeatures = featureDefinition.getNumberOfFeatures(); + String[] featureNames = new String[noOfFeatures]; + for (int i = 0; i < featureNames.length; i++) { + featureNames[i] = featureDefinition.getFeatureName(i); + } + + for (int i = 0; i < bnlVocalizations.getLength(); i++) { + + byte[] byteFeatures; + short[] shortFeatures; + float[] continiousFeatures; + + String baseName = bnlVocalizations.getName(i); + Map singleAnnotation = annotationData.get(baseName); + + int noByteFeatures = featureDefinition.getNumberOfByteFeatures(); + int noShortFeatures = featureDefinition.getNumberOfShortFeatures(); + int noContiniousFeatures = featureDefinition.getNumberOfContinuousFeatures(); + + // create features array + byteFeatures = new byte[noByteFeatures]; + shortFeatures = new short[noShortFeatures]; + continiousFeatures = new float[noContiniousFeatures]; + + int countByteFeatures = 0; + int countShortFeatures = 0; + int countFloatFeatures = 0; + + for (int j = 0; j < featureNames.length; j++) { + String fName = featureNames[j]; + if (featureDefinition.isByteFeature(j)) { // Byte feature + if (singleAnnotation.containsKey(fName)) { + byteFeatures[countByteFeatures++] = featureDefinition.getFeatureValueAsByte(j, + singleAnnotation.get(fName)); + } else { + byteFeatures[countByteFeatures++] = (byte) 0; + } + } else if (featureDefinition.isShortFeature(j)) { // Short feature + if (singleAnnotation.containsKey(fName)) { + shortFeatures[countShortFeatures++] = featureDefinition.getFeatureValueAsShort(j, + singleAnnotation.get(fName)); + } else { + shortFeatures[countShortFeatures++] = (short) 0; + } + } else if (featureDefinition.isContinuousFeature(j)) { // Continuous feature + if (!singleAnnotation.containsKey(fName)) { + continiousFeatures[countFloatFeatures++] = Float.NaN; + } else if ("NRI".equals(singleAnnotation.get(fName))) { + continiousFeatures[countFloatFeatures++] = Float.NaN; + } else { + continiousFeatures[countFloatFeatures++] = (new Float(singleAnnotation.get(fName))).floatValue(); + } + } + } + + FeatureVector outFV = featureDefinition.toFeatureVector(i, byteFeatures, shortFeatures, continiousFeatures); + outFV.writeTo(out); + } + + } + + /** + * Write the header of this feature file to the given DataOutput + * + * @param out + * @throws IOException + */ + protected void writeHeaderTo(DataOutput out) throws IOException { + new MaryHeader(MaryHeader.LISTENERFEATS).writeTo(out); + featureDefinition.writeBinaryTo(out); + } + + /** + * Provide the progress of computation, in percent, or -1 if that feature is not implemented. + * + * @return -1 if not implemented, or an integer between 0 and 100. + */ + public int getProgress() { + return percent; + } + + /** + * @param args + */ + public static void main(String[] args) throws Exception { + VocalizationFeatureFileWriter acfeatsWriter = new VocalizationFeatureFileWriter(); + DatabaseLayout db = new DatabaseLayout(acfeatsWriter); + acfeatsWriter.compute(); + } + +} diff --git a/marytts-purgatory/marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/VocalizationIntonationWriter.java b/marytts-purgatory/marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/VocalizationIntonationWriter.java new file mode 100644 index 0000000000..590f56e031 --- /dev/null +++ b/marytts-purgatory/marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/VocalizationIntonationWriter.java @@ -0,0 +1,435 @@ +/** + * Copyright 2010 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ + +package marytts.tools.voiceimport.vocalizations; + +import java.io.BufferedOutputStream; +import java.io.DataOutput; +import java.io.DataOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.Arrays; +import java.util.SortedMap; +import java.util.TreeMap; + +import javax.sound.sampled.AudioFormat; +import javax.sound.sampled.AudioInputStream; +import javax.sound.sampled.AudioSystem; +import javax.sound.sampled.UnsupportedAudioFileException; + +import marytts.exceptions.MaryConfigurationException; +import marytts.signalproc.analysis.F0TrackerAutocorrelationHeuristic; +import marytts.signalproc.analysis.PitchFileHeader; +import marytts.signalproc.analysis.PitchReaderWriter; +import marytts.signalproc.analysis.SPTKPitchReaderWriter; +import marytts.tools.voiceimport.DatabaseLayout; +import marytts.tools.voiceimport.VoiceImportComponent; +import marytts.util.data.BufferedDoubleDataSource; +import marytts.util.data.MaryHeader; +import marytts.util.data.audio.AudioDoubleDataSource; +import marytts.util.io.BasenameList; +import marytts.util.math.Polynomial; +import marytts.util.signal.SignalProcUtils; +import marytts.vocalizations.VocalizationIntonationReader; +import marytts.vocalizations.VocalizationUnitFileReader; + +/** + * Vocalization intonation writer into a time-line file This class can create a timeline file with intonation contours and thier + * polynomial coeffs + * + * @author sathish pammi + * + */ +public class VocalizationIntonationWriter extends VoiceImportComponent { + + protected String vocalizationsDir; + protected BasenameList bnlVocalizations; + protected VocalizationUnitFileReader listenerUnits; + + protected DatabaseLayout db = null; + protected int percent = 0; + + public final String PITCHDIR = getName() + ".pitchDir"; + public final String WAVEDIR = getName() + ".inputWaveDir"; + public final String POLYORDER = getName() + ".polynomialOrder"; + public final String ISEXTERNALF0 = getName() + ".isExternalF0Usage"; + public final String EXTERNALF0FORMAT = getName() + ".externalF0Format"; + public final String EXTERNALEXT = getName() + ".externalF0Extention"; + public final String UNITFILE = getName() + ".unitFile"; + public final String SKIPSIZE = getName() + ".skipSize"; + public final String WINDOWSIZE = getName() + ".windowSize"; + public final String F0TIMELINE = getName() + ".intonationTimeLineFile"; + public final String F0FEATDEF = getName() + ".intonationFeatureDefinition"; + + public String getName() { + return "VocalizationIntonationWriter"; + } + + @Override + protected void initialiseComp() { + + String timelineDir = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "files"; + if (!(new File(timelineDir)).exists()) { + System.out.println("vocalizations/files directory does not exist; "); + if (!(new File(timelineDir)).mkdirs()) { + throw new Error("Could not create vocalizations/files"); + } + System.out.println("Created successfully.\n"); + } + + try { + String basenameFile = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "basenames.lst"; + if ((new File(basenameFile)).exists()) { + System.out.println("Loading basenames of vocalizations from '" + basenameFile + "' list..."); + bnlVocalizations = new BasenameList(basenameFile); + System.out.println("Found " + bnlVocalizations.getLength() + " vocalizations in basename list"); + } else { + String vocalWavDir = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "wav"; + System.out.println("Loading basenames of vocalizations from '" + vocalWavDir + "' directory..."); + bnlVocalizations = new BasenameList(vocalWavDir, ".wav"); + System.out.println("Found " + bnlVocalizations.getLength() + " vocalizations in " + vocalWavDir + " directory"); + } + } catch (IOException e) { + e.printStackTrace(); + } + } + + public SortedMap getDefaultProps(DatabaseLayout db) { + this.db = db; + if (props == null) { + props = new TreeMap(); + props.put(WAVEDIR, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "wav"); + props.put(UNITFILE, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "files" + File.separator + + "vocalization_units" + db.getProp(db.MARYEXT)); + props.put(POLYORDER, "3"); + props.put(ISEXTERNALF0, "true"); + props.put(EXTERNALF0FORMAT, "sptk"); + props.put(EXTERNALEXT, ".lf0"); + props.put(PITCHDIR, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "lf0"); + props.put(SKIPSIZE, "0.005"); + props.put(WINDOWSIZE, "0.005"); + props.put(F0TIMELINE, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "files" + File.separator + + "vocalization_intonation" + db.getProp(db.MARYEXT)); + props.put(F0FEATDEF, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "features" + File.separator + + "vocalization_f0_feature_definition.txt"); + } + return props; + } + + protected void setupHelp() { + props2Help = new TreeMap(); + + } + + /** + * Reads and concatenates a list of waveforms into one single timeline file. + * + * @throws IOException + */ + @Override + public boolean compute() throws IOException, MaryConfigurationException { + + listenerUnits = new VocalizationUnitFileReader(getProp(UNITFILE)); + + // write features into timeline file + DataOutputStream out = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(new File(getProp(F0TIMELINE))))); + writeHeaderTo(out); + writeUnitFeaturesTo(out); + out.close(); + + VocalizationIntonationReader tester = new VocalizationIntonationReader(getProp(F0TIMELINE)); + int unitsOnDisk = tester.getNumberOfUnits(); + if (unitsOnDisk == listenerUnits.getNumberOfUnits()) { + System.out.println("Can read right number of units"); + return true; + } else { + System.out.println("Read wrong number of units: " + unitsOnDisk); + return false; + } + } + + /** + * + * @param out + * @throws IOException + */ + protected void writeUnitFeaturesTo(DataOutput out) throws IOException { + + int numUnits = listenerUnits.getNumberOfUnits(); + float windowSize = new Float(getProp(WINDOWSIZE)).floatValue(); + float skipSize = new Float(getProp(SKIPSIZE)).floatValue(); + + out.writeFloat(windowSize); + out.writeFloat(skipSize); + out.writeInt(numUnits); + + for (int i = 0; i < bnlVocalizations.getLength(); i++) { + + double[] f0Array = null; + + try { + f0Array = getVocalizationF0(bnlVocalizations.getName(i), false); + } catch (UnsupportedAudioFileException e) { + e.printStackTrace(); + } + + // write coeffs followed by its order + double[] coeffs = getPolynomialCoeffs(f0Array); + if (coeffs == null) { + out.writeInt(0); + } else { + out.writeInt(coeffs.length); + for (int j = 0; j < coeffs.length; j++) { + out.writeFloat((float) coeffs[j]); + } + } + + // write f0 Array followed by f0 contour array size + if (f0Array == null) { + out.writeInt(0); + } else { + out.writeInt(f0Array.length); + for (int j = 0; j < f0Array.length; j++) { + out.writeFloat((float) f0Array[j]); + } + } + } + } + + /** + * get f0 contour of vocalization f0 + * + * @param baseName + * @return + * @throws UnsupportedAudioFileException + * @throws IOException + */ + private double[] getVocalizationF0(String baseName, boolean doInterpolate) throws UnsupportedAudioFileException, IOException { + + double[] f0Array = null; + + if ("true".equals(getProp(ISEXTERNALF0))) { + + String externalFormat = getProp(EXTERNALF0FORMAT); + String externalExt = getProp(EXTERNALEXT); + System.out.println("Loading f0 contour from file : " + getProp(PITCHDIR) + File.separator + baseName + externalExt); + if ("sptk".equals(externalFormat)) { + String fileName = getProp(PITCHDIR) + File.separator + baseName + externalExt; + SPTKPitchReaderWriter sprw = new SPTKPitchReaderWriter(fileName); + f0Array = sprw.getF0Contour(); + } else if ("ptc".equals(externalFormat)) { + String fileName = getProp(PITCHDIR) + File.separator + baseName + externalExt; + PitchReaderWriter sprw = new PitchReaderWriter(fileName); + f0Array = sprw.contour; + } + } else { + PitchFileHeader params = new PitchFileHeader(); + F0TrackerAutocorrelationHeuristic tracker = new F0TrackerAutocorrelationHeuristic(params); + String waveFile = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "wav" + baseName + db.getProp(db.WAVEXT); + System.out.println("Computing f0 contour from wave file: " + waveFile); + AudioInputStream inputAudio = AudioSystem.getAudioInputStream(new File(waveFile)); + + // Enforce PCM_SIGNED encoding + if (!inputAudio.getFormat().getEncoding().equals(AudioFormat.Encoding.PCM_SIGNED)) { + inputAudio = AudioSystem.getAudioInputStream(AudioFormat.Encoding.PCM_SIGNED, inputAudio); + } + + int audioSampleRate = (int) inputAudio.getFormat().getSampleRate(); + AudioDoubleDataSource signal = new AudioDoubleDataSource(inputAudio); + double[] sentenceAudio = signal.getAllData(); + tracker.pitchAnalyze(new BufferedDoubleDataSource(sentenceAudio)); + // double frameShiftTime = tracker.getSkipSizeInSeconds(); + f0Array = tracker.getF0Contour(); + } + + if (doInterpolate) { + return interpolateF0Array(f0Array); + } + + return f0Array; + } + + /** + * to get polynomial coeffs of f0 contour + * + * @param f0Array + * @return + */ + private double[] getPolynomialCoeffs(double[] f0Array) { + + if (f0Array == null) { + return null; + } + + f0Array = cutStartEndUnvoicedSegments(f0Array); + double[] f0AndInterpolate = interpolateF0Array(f0Array); + int polynomialOrder = (new Integer(getProp(POLYORDER))).intValue(); + double[] coeffs = Polynomial.fitPolynomial(f0AndInterpolate, polynomialOrder); + return coeffs; + } + + /** + * to interpolate F0 contour values + * + * @param f0Array + * @return + */ + private double[] interpolateF0Array(double[] f0Array) { + + if (f0Array == null) { + return null; + } + + for (int j = 0; j < f0Array.length; j++) { + if (f0Array[j] == 0) { + f0Array[j] = Double.NaN; + } + } + if (f0Array.length >= 3) { + f0Array = SignalProcUtils.medianFilter(f0Array, 5); + } + + double[] f0AndInterpol; + double[] interpol = new double[f0Array.length]; + Arrays.fill(interpol, Double.NaN); + f0AndInterpol = new double[f0Array.length]; + int iLastValid = -1; + for (int j = 0; j < f0Array.length; j++) { + if (!Double.isNaN(f0Array[j])) { // a valid value + if (iLastValid == j - 1) { + // no need to interpolate + f0AndInterpol[j] = f0Array[j]; + } else { + // need to interpolate + double prevF0; + if (iLastValid < 0) { // we don't have a previous value -- use current one + prevF0 = f0Array[j]; + } else { + prevF0 = f0Array[iLastValid]; + } + double delta = (f0Array[j] - prevF0) / (j - iLastValid); + double f0 = prevF0; + for (int k = iLastValid + 1; k < j; k++) { + f0 += delta; + interpol[k] = f0; + f0AndInterpol[k] = f0; + } + } + iLastValid = j; + } + } + + double[] f0AndInterpolate = combineF0andInterpolate(f0Array, interpol); + return f0AndInterpolate; + } + + /** + * cut begin-end unvoiced segments + * + * @param array + * @return + */ + private double[] cutStartEndUnvoicedSegments(double[] array) { + + if (array == null) + return null; + + int startIndex = 0; + int endIndex = array.length; + + // find start index + for (int i = 0; i < array.length; i++) { + if (array[i] != 0) { + startIndex = i; + break; + } + } + + // find end index + for (int i = (array.length - 1); i > 0; i--) { + if (array[i] != 0) { + endIndex = i; + break; + } + } + + int newArraySize = endIndex - startIndex; + + double[] newArray = new double[newArraySize]; + System.arraycopy(array, startIndex, newArray, 0, newArraySize); + + /* + * for ( int i=0; i. + * + */ + +package marytts.tools.voiceimport.vocalizations; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.io.PrintWriter; +import java.util.SortedMap; +import java.util.TreeMap; + +import marytts.tools.voiceimport.DatabaseLayout; +import marytts.tools.voiceimport.PraatPitchmarker; +import marytts.util.io.BasenameList; + +public class VocalizationPitchmarker extends PraatPitchmarker { + + String vocalizationsDir; + BasenameList bnlVocalizations; + + public String getName() { + return "VocalizationPitchmarker"; + } + + @Override + protected void initialiseComp() { + tmpScript = db.getProp(db.TEMPDIR) + "script.praat"; + + if (!(new File(getProp(PRAATPMDIR))).exists()) { + + System.out.println("vocalizations/pm directory does not exist; "); + if (!(new File(getProp(PRAATPMDIR))).mkdirs()) { + throw new Error("Could not create vocalizations/pm"); + } + System.out.println("Created successfully.\n"); + + } + + try { + String basenameFile = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "basenames.lst"; + if ((new File(basenameFile)).exists()) { + System.out.println("Loading basenames of vocalizations from '" + basenameFile + "' list..."); + bnlVocalizations = new BasenameList(basenameFile); + System.out.println("Found " + bnlVocalizations.getLength() + " vocalizations in basename list"); + } else { + String vocalWavDir = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "wav"; + System.out.println("Loading basenames of vocalizations from '" + vocalWavDir + "' directory..."); + bnlVocalizations = new BasenameList(vocalWavDir, ".wav"); + System.out.println("Found " + bnlVocalizations.getLength() + " vocalizations in " + vocalWavDir + " directory"); + } + } catch (IOException e) { + e.printStackTrace(); + } + } + + public SortedMap getDefaultProps(DatabaseLayout db) { + this.db = db; + if (props == null) { + props = new TreeMap(); + props.put(COMMAND, "praat"); + if (db.getProp(db.GENDER).equals("female")) { + props.put(MINPITCH, "100"); + props.put(MAXPITCH, "500"); + } else { + props.put(MINPITCH, "75"); + props.put(MAXPITCH, "300"); + } + props.put(WAVEDIR, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "wav"); + props.put(PRAATPMDIR, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "pm"); + // vocalizationsDir = db.getProp(db.ROOTDIR)+File.separator+"vocalizations"; + } + return props; + } + + /** + * The standard compute() method of the VoiceImportComponent interface. + */ + public boolean compute() throws IOException { + + String[] baseNameArray = bnlVocalizations.getListAsArray(); + System.out.println("Computing pitchmarks for " + baseNameArray.length + " utterances."); + + /* Ensure the existence of the target pitchmark directory */ + File dir = new File(getProp(PRAATPMDIR)); + if (!dir.exists()) { + System.out.println("Creating the directory [" + getProp(PRAATPMDIR) + "]."); + dir.mkdirs(); + } + + // script.praat is provided as template. Perhaps it could be used instead of hardcoding the following: + File script = new File(tmpScript); + + if (script.exists()) + script.delete(); + PrintWriter toScript = new PrintWriter(new FileWriter(script)); + // use Praat form to provide ARGV (NOTE: these must be explicitly given during call to Praat!) + toScript.println("form Provide arguments"); + toScript.println(" sentence wavFile input.wav"); + toScript.println(" sentence pointpFile output.PointProcess"); + toScript.println(" real minPitch 75"); + toScript.println(" real maxPitch 600"); + toScript.println("endform"); + toScript.println("Read from file... 'wavFile$'"); + // Remove DC offset, if present: + toScript.println("Subtract mean"); + // First, low-pass filter the speech signal to make it more robust against noise + // (i.e., mixed noise+periodicity regions treated more likely as periodic) + toScript.println("sound = Filter (pass Hann band)... 0 1000 100"); + // Then determine pitch curve: + toScript.println("pitch = To Pitch... 0 minPitch maxPitch"); + // Get some debug info: + toScript.println("min_f0 = Get minimum... 0 0 Hertz Parabolic"); + toScript.println("max_f0 = Get maximum... 0 0 Hertz Parabolic"); + // And convert to pitch marks: + toScript.println("plus sound"); + toScript.println("To PointProcess (cc)"); + // Fill in 100 Hz pseudo-pitchmarks in unvoiced regions: + toScript.println("Voice... 0.01 0.02000000001"); + toScript.println("Write to short text file... 'pointpFile$'"); + toScript.println("lastSlash = rindex(wavFile$, \"/\")"); + toScript.println("baseName$ = right$(wavFile$, length(wavFile$) - lastSlash) - \".wav\""); + toScript.println("printline 'baseName$' f0 range: 'min_f0:0' - 'max_f0:0' Hz"); + toScript.close(); + + System.out.println("Running Praat as: " + getProp(COMMAND) + " " + tmpScript); + for (int i = 0; i < baseNameArray.length; i++) { + percent = 100 * i / baseNameArray.length; + praatPitchmarks(baseNameArray[i]); + } + + return true; + } + + /** + * @param args + */ + public static void main(String[] args) { + // TODO Auto-generated method stub + + } + +} diff --git a/marytts-purgatory/marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/VocalizationTimelineMaker.java b/marytts-purgatory/marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/VocalizationTimelineMaker.java new file mode 100644 index 0000000000..48d236103e --- /dev/null +++ b/marytts-purgatory/marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/VocalizationTimelineMaker.java @@ -0,0 +1,233 @@ +/** + * Copyright 2010 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ + +package marytts.tools.voiceimport.vocalizations; + +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.io.File; +import java.io.IOException; +import java.text.DecimalFormat; +import java.util.SortedMap; +import java.util.TreeMap; + +import marytts.tools.voiceimport.DatabaseLayout; +import marytts.tools.voiceimport.TimelineWriter; +import marytts.tools.voiceimport.VoiceImportComponent; +import marytts.tools.voiceimport.WavReader; +import marytts.util.data.Datagram; +import marytts.util.data.ESTTrackReader; +import marytts.util.io.BasenameList; + +public class VocalizationTimelineMaker extends VoiceImportComponent { + + String vocalizationsDir; + BasenameList bnlVocalizations; + + protected DatabaseLayout db = null; + protected int percent = 0; + public final String WAVETIMELINE = "VocalizationTimelineMaker.waveTimeline"; + public final String WAVEDIR = "VocalizationTimelineMaker.inputWaveDir"; + public final String PMARKDIR = "VocalizationTimelineMaker.pitchmarkDir"; + + public final String PMDIR = "db.pmDir"; + public final String PMEXT = "db.pmExtension"; + + public String getName() { + return "VocalizationTimelineMaker"; + } + + @Override + protected void initialiseComp() { + String timelineDir = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "files"; + if (!(new File(timelineDir)).exists()) { + + System.out.println("vocalizations/files directory does not exist; "); + if (!(new File(timelineDir)).mkdirs()) { + throw new Error("Could not create vocalizations/files"); + } + System.out.println("Created successfully.\n"); + + } + + try { + String basenameFile = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "basenames.lst"; + if ((new File(basenameFile)).exists()) { + System.out.println("Loading basenames of vocalizations from '" + basenameFile + "' list..."); + bnlVocalizations = new BasenameList(basenameFile); + System.out.println("Found " + bnlVocalizations.getLength() + " vocalizations in basename list"); + } else { + String vocalWavDir = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "wav"; + System.out.println("Loading basenames of vocalizations from '" + vocalWavDir + "' directory..."); + bnlVocalizations = new BasenameList(vocalWavDir, ".wav"); + System.out.println("Found " + bnlVocalizations.getLength() + " vocalizations in " + vocalWavDir + " directory"); + } + } catch (IOException e) { + e.printStackTrace(); + } + } + + public SortedMap getDefaultProps(DatabaseLayout db) { + this.db = db; + if (props == null) { + props = new TreeMap(); + props.put(WAVEDIR, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "wav"); + props.put(PMARKDIR, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "pm"); + props.put(WAVETIMELINE, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "files" + File.separator + + "vocalization_wave_timeline" + db.getProp(db.MARYEXT)); + // vocalizationsDir = db.getProp(db.ROOTDIR)+File.separator+"vocalizations"; + } + return props; + } + + protected void setupHelp() { + props2Help = new TreeMap(); + props2Help.put(WAVETIMELINE, "file containing all wave files. Will be created by this module"); + } + + /** + * Reads and concatenates a list of waveforms into one single timeline file. + * + */ + public boolean compute() { + System.out.println("---- Making a pitch synchronous waveform timeline\n\n"); + + /* Export the basename list into an array of strings */ + String[] baseNameArray = bnlVocalizations.getListAsArray(); + System.out.println("Processing [" + baseNameArray.length + "] utterances.\n"); + + try { + /* + * 1) Determine the reference sampling rate as being the sample rate of the first encountered wav file + */ + WavReader wav = new WavReader(getProp(WAVEDIR) + baseNameArray[0] + db.getProp(db.WAVEXT)); + int globSampleRate = wav.getSampleRate(); + System.out.println("---- Detected a global sample rate of: [" + globSampleRate + "] Hz."); + + System.out.println("---- Folding the wav files according to the pitchmarks..."); + + /* 2) Open the destination timeline file */ + + /* Make the file name */ + String waveTimelineName = getProp(WAVETIMELINE); + System.out.println("Will create the waveform timeline in file [" + waveTimelineName + "]."); + + /* Processing header: */ + String processingHeader = "\n"; + + /* Instantiate the TimelineWriter: */ + TimelineWriter waveTimeline = new TimelineWriter(waveTimelineName, processingHeader, globSampleRate, 0.1); + + /* 3) Write the datagrams and feed the index */ + + float totalDuration = 0.0f; // Accumulator for the total timeline duration + long totalTime = 0l; + int numDatagrams = 0; + + /* For each EST track file: */ + ESTTrackReader pmFile = null; + for (int i = 0; i < baseNameArray.length; i++) { + percent = 100 * i / baseNameArray.length; + + /* - open+load */ + System.out.println(baseNameArray[i]); + pmFile = new ESTTrackReader(getProp(PMARKDIR) + baseNameArray[i] + db.getProp(PMEXT)); + totalDuration += pmFile.getTimeSpan(); + wav = new WavReader(getProp(WAVEDIR) + baseNameArray[i] + db.getProp(db.WAVEXT)); + short[] wave = wav.getSamples(); + /* - Reset the frame locations in the local file */ + int frameStart = 0; + int frameEnd = 0; + int duration = 0; + long localTime = 0l; + /* - For each frame in the WAV file: */ + for (int f = 0; f < pmFile.getNumFrames(); f++) { + + /* Locate the corresponding segment in the wave file */ + frameStart = frameEnd; + frameEnd = (int) ((double) pmFile.getTime(f) * (double) (globSampleRate)); + assert frameEnd <= wave.length : "Frame ends after end of wave data: " + frameEnd + " > " + wave.length; + + duration = frameEnd - frameStart; + ByteArrayOutputStream buff = new ByteArrayOutputStream(2 * duration); + DataOutputStream subWave = new DataOutputStream(buff); + for (int k = 0; k < duration; k++) { + subWave.writeShort(wave[frameStart + k]); + } + + // Handle the case when the last pitch marks falls beyond the end of the signal + + /* Feed the datagram to the timeline */ + waveTimeline.feed(new Datagram(duration, buff.toByteArray()), globSampleRate); + totalTime += duration; + localTime += duration; + numDatagrams++; + } + // System.out.println( baseNameArray[i] + " -> pm file says [" + localTime + "] samples, wav file says ["+ + // wav.getNumSamples() + "] samples." ); + } + waveTimeline.close(); + + System.out.println("---- Done."); + + /* 7) Print some stats and close the file */ + System.out.println("---- Waveform timeline result:"); + System.out.println("Number of files scanned: " + baseNameArray.length); + System.out.println("Total speech duration: [" + totalTime + "] samples / [" + + ((float) (totalTime) / (float) (globSampleRate)) + "] seconds."); + System.out.println("(Speech duration approximated from EST Track float times: [" + totalDuration + "] seconds.)"); + System.out.println("Number of frames: [" + numDatagrams + "]."); + System.out.println("Size of the index: [" + waveTimeline.getIndex().getNumIdx() + "] (" + + (waveTimeline.getIndex().getNumIdx() * 16) + " bytes, i.e. " + + new DecimalFormat("#.##").format((double) (waveTimeline.getIndex().getNumIdx()) * 16.0 / 1048576.0) + + " megs)."); + System.out.println("---- Waveform timeline done."); + + } catch (SecurityException e) { + System.err.println("Error: you don't have write access to the target database directory."); + e.printStackTrace(); + return false; + } catch (Exception e) { + e.printStackTrace(); + System.err.println(e); + return false; + } + + return (true); + } + + /** + * Provide the progress of computation, in percent, or -1 if that feature is not implemented. + * + * @return -1 if not implemented, or an integer between 0 and 100. + */ + public int getProgress() { + return percent; + } + + /** + * @param args + */ + public static void main(String[] args) { + // TODO Auto-generated method stub + + } + +} diff --git a/marytts-purgatory/marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/VocalizationUnitfileWriter.java b/marytts-purgatory/marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/VocalizationUnitfileWriter.java new file mode 100644 index 0000000000..343f39a878 --- /dev/null +++ b/marytts-purgatory/marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/VocalizationUnitfileWriter.java @@ -0,0 +1,319 @@ +/** + * Copyright 2006 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.tools.voiceimport.vocalizations; + +import java.io.BufferedOutputStream; +import java.io.BufferedReader; +import java.io.DataOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.SortedMap; +import java.util.StringTokenizer; +import java.util.TreeMap; + +import marytts.exceptions.MaryConfigurationException; +import marytts.tools.voiceimport.DatabaseLayout; +import marytts.tools.voiceimport.VoiceImportComponent; +import marytts.util.data.ESTTrackReader; +import marytts.util.data.MaryHeader; +import marytts.util.io.BasenameList; +import marytts.vocalizations.VocalizationUnitFileReader; + +/** + * Back-channel unit writer + * + * @author sathish pammi + * + */ +public class VocalizationUnitfileWriter extends VoiceImportComponent { + + protected String vocalizationsDir; + protected BasenameList bnlVocalizations; + + // protected File maryDir; + protected String unitFileName; + protected File unitlabelDir; + protected int samplingRate; + // protected String pauseSymbol; + protected String unitlabelExt = ".lab"; + + protected DatabaseLayout db = null; + protected int percent = 0; + // protected BasenameList bachChannelList; + + public String LABELDIR = "VocalizationUnitfileWriter.backchannelLabDir"; + public String UNITFILE = "VocalizationUnitfileWriter.unitFile"; + public final String PMARKDIR = "VocalizationUnitfileWriter.pitchmarkDir"; + // public String BASELIST = "VocalizationUnitfileWriter.backchannelBaseNamesList"; + + public final String PMDIR = "db.pmDir"; + public final String PMEXT = "db.pmExtension"; + + public String getName() { + return "VocalizationUnitfileWriter"; + } + + @Override + protected void initialiseComp() { + // maryDir = new File(db.getProp(db.FILEDIR)); + + samplingRate = Integer.parseInt(db.getProp(db.SAMPLINGRATE)); + // pauseSymbol = System.getProperty("pause.symbol", "pau"); + + unitFileName = getProp(UNITFILE); + unitlabelDir = new File(getProp(LABELDIR)); + + String timelineDir = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "files"; + if (!(new File(timelineDir)).exists()) { + + System.out.println("vocalizations/files directory does not exist; "); + if (!(new File(timelineDir)).mkdirs()) { + throw new Error("Could not create vocalizations/files"); + } + System.out.println("Created successfully.\n"); + + } + + try { + String basenameFile = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "basenames.lst"; + if ((new File(basenameFile)).exists()) { + System.out.println("Loading basenames of vocalizations from '" + basenameFile + "' list..."); + bnlVocalizations = new BasenameList(basenameFile); + System.out.println("Found " + bnlVocalizations.getLength() + " vocalizations in basename list"); + } else { + String vocalWavDir = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "wav"; + System.out.println("Loading basenames of vocalizations from '" + vocalWavDir + "' directory..."); + bnlVocalizations = new BasenameList(vocalWavDir, ".wav"); + System.out.println("Found " + bnlVocalizations.getLength() + " vocalizations in " + vocalWavDir + " directory"); + } + } catch (IOException e) { + e.printStackTrace(); + } + + } + + public SortedMap getDefaultProps(DatabaseLayout db) { + this.db = db; + if (props == null) { + props = new TreeMap(); + String vocalizationsrootDir = db.getProp(db.VOCALIZATIONSDIR); + props.put(LABELDIR, vocalizationsrootDir + File.separator + "lab" + System.getProperty("file.separator")); + props.put( + UNITFILE, + vocalizationsrootDir + File.separator + "files" + File.separator + "vocalization_units" + + db.getProp(db.MARYEXT)); + props.put(PMARKDIR, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "pm"); + // props.put(BASELIST, "backchannel.lst"); + } + return props; + } + + protected void setupHelp() { + props2Help = new TreeMap(); + props2Help.put(LABELDIR, "directory containing the phone labels"); + props2Help.put(UNITFILE, "file containing all phone units. Will be created by this module"); + } + + @Override + public boolean compute() throws IOException, MaryConfigurationException { + if (!unitlabelDir.exists()) { + System.out.print(LABELDIR + " " + getProp(LABELDIR) + " does not exist; "); + throw new Error("LABELDIR not found"); + } + + System.out.println("Back channel unitfile writer started..."); + BackChannelUnits bcUnits = new BackChannelUnits(unitlabelDir.getAbsolutePath(), this.bnlVocalizations); + DataOutputStream out = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(unitFileName))); + long posNumUnits = new MaryHeader(MaryHeader.LISTENERUNITS).writeTo(out); + int numberOfBCUnits = bcUnits.getNumberOfUnits(); + out.writeInt(numberOfBCUnits); + out.writeInt(samplingRate); + long globalStart = 0l; // time, given as sample position with samplingRate + for (int i = 0; i < numberOfBCUnits; i++) { + UnitLabel[] fileLabels = bcUnits.getUnitLabels(i); + double unitTimeSpan = bcUnits.getTimeSpan(i); + int localLabLength = fileLabels.length; + out.writeInt(localLabLength); + for (int j = 0; j < localLabLength; j++) { + double startTime = fileLabels[j].startTime; + double endTime = fileLabels[j].endTime; + double duration = endTime - startTime; + long end = (long) (endTime * (double) (samplingRate)); + long start = (long) (startTime * (double) (samplingRate)); + out.writeLong(globalStart + start); + out.writeInt((int) (end - start)); + out.writeInt(fileLabels[j].unitName.toCharArray().length); + out.writeChars(fileLabels[j].unitName); + } + globalStart += ((long) ((double) (unitTimeSpan) * (double) (samplingRate))); + } + + out.close(); + VocalizationUnitFileReader tester = new VocalizationUnitFileReader(unitFileName); + int unitsOnDisk = tester.getNumberOfUnits(); + if (unitsOnDisk == numberOfBCUnits) { + System.out.println("Can read right number of units: " + unitsOnDisk); + return true; + } else { + System.out.println("Read wrong number of units: " + unitsOnDisk); + return false; + } + } + + class BackChannelUnits { + int numberOfUnits; + UnitLabel[][] unitLabels; + double[] unitTimeSpan; + + BackChannelUnits(String unitlabelDir, BasenameList basenameList) throws IOException { + this.numberOfUnits = basenameList.getLength(); + unitLabels = new UnitLabel[this.numberOfUnits][]; + unitTimeSpan = new double[this.numberOfUnits]; + for (int i = 0; i < this.numberOfUnits; i++) { + String fileName = unitlabelDir + File.separator + basenameList.getName(i) + unitlabelExt; + ESTTrackReader pmFile = new ESTTrackReader(getProp(PMARKDIR) + File.separator + basenameList.getName(i) + + db.getProp(PMEXT)); + unitLabels[i] = readLabFile(fileName); + unitTimeSpan[i] = pmFile.getTimeSpan(); + } + } + + int getNumberOfUnits() { + return this.numberOfUnits; + } + + UnitLabel[] getUnitLabels(int i) { + return this.unitLabels[i]; + } + + double getTimeSpan(int i) { + return this.unitTimeSpan[i]; + } + } + + class UnitLabel { + String unitName; + double startTime; + double endTime; + int unitIndex; + + public UnitLabel(String unitName, double startTime, double endTime, int unitIndex) { + this.unitName = unitName; + this.startTime = startTime; + this.endTime = endTime; + this.unitIndex = unitIndex; + } + } + + /** + * @param labFile + * @return + */ + private UnitLabel[] readLabFile(String labFile) throws IOException { + + ArrayList lines = new ArrayList(); + BufferedReader labels = new BufferedReader(new InputStreamReader(new FileInputStream(new File(labFile)), "UTF-8")); + String line; + + // Read Label file first + // 1. Skip label file header: + while ((line = labels.readLine()) != null) { + if (line.startsWith("#")) + break; // line starting with "#" marks end of header + } + + // 2. Put data into an ArrayList + String labelUnit = null; + double startTimeStamp = 0.0; + double endTimeStamp = 0.0; + int unitIndex = 0; + while ((line = labels.readLine()) != null) { + labelUnit = null; + if (line != null) { + List labelUnitData = getLabelUnitData(line); + if (labelUnitData == null) + continue; + labelUnit = (String) labelUnitData.get(2); + unitIndex = Integer.parseInt((String) labelUnitData.get(1)); + endTimeStamp = Double.parseDouble((String) labelUnitData.get(0)); + } + if (labelUnit == null) + break; + lines.add(labelUnit.trim() + " " + startTimeStamp + " " + endTimeStamp + " " + unitIndex); + startTimeStamp = endTimeStamp; + } + labels.close(); + + UnitLabel[] ulab = new UnitLabel[lines.size()]; + Iterator itr = lines.iterator(); + for (int i = 0; itr.hasNext(); i++) { + String element = itr.next(); + String[] wrds = element.split("\\s+"); + ulab[i] = new UnitLabel(wrds[0], (new Double(wrds[1])).doubleValue(), (new Double(wrds[2])).doubleValue(), + (new Integer(wrds[3])).intValue()); + } + return ulab; + } + + /** + * To get Label Unit DATA (time stamp, index, phone unit) + * + * @param line + * @return ArrayList contains time stamp, index and phone unit + * @throws IOException + */ + private ArrayList getLabelUnitData(String line) throws IOException { + if (line == null) + return null; + if (line.trim().equals("")) + return null; + ArrayList unitData = new ArrayList(); + StringTokenizer st = new StringTokenizer(line.trim()); + // the first token is the time + unitData.add(st.nextToken()); + // the second token is the unit index + unitData.add(st.nextToken()); + // the third token is the phone + unitData.add(st.nextToken()); + return unitData; + } + + /** + * Provide the progress of computation, in percent, or -1 if that feature is not implemented. + * + * @return -1 if not implemented, or an integer between 0 and 100. + */ + public int getProgress() { + return percent; + } + + public static void main(String[] args) throws Exception { + VocalizationUnitfileWriter ufw = new VocalizationUnitfileWriter(); + new DatabaseLayout(ufw); + ufw.compute(); + } + +} diff --git a/marytts-purgatory/marytts-builder/src/main/java/marytts/vocalizations/VocalisationLabelInspector.java b/marytts-purgatory/marytts-builder/src/main/java/marytts/vocalizations/VocalisationLabelInspector.java new file mode 100644 index 0000000000..c258075c31 --- /dev/null +++ b/marytts-purgatory/marytts-builder/src/main/java/marytts/vocalizations/VocalisationLabelInspector.java @@ -0,0 +1,252 @@ +/** + * Copyright 2010 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.vocalizations; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; + +import javax.sound.sampled.AudioFormat; +import javax.sound.sampled.AudioInputStream; +import javax.sound.sampled.AudioSystem; + +import marytts.tools.voiceimport.UnitLabel; +import marytts.util.data.audio.MaryAudioUtils; +import marytts.util.io.BasenameList; + +public class VocalisationLabelInspector { + + private String inLocation; // input directory + private String outLocation; // output directory + private AudioFormat format; + + /** + * + * @param inLocation + * @param outLocation + */ + public VocalisationLabelInspector(String inLocation, String outLocation) { + this.inLocation = inLocation; + this.outLocation = outLocation; + } + + public void process(String baseName) throws IOException { + + String labelFile = inLocation + File.separator + baseName + ".lab"; + String outlabelFile = outLocation + File.separator + baseName + ".lab"; + String waveFile = inLocation + File.separator + baseName + ".wav"; + String outwaveFile = outLocation + File.separator + baseName + ".wav"; + + // read labels + UnitLabel[] vocalLabels = UnitLabel.readLabFile(labelFile); + + // read waveform + AudioInputStream audioInputStream = null; + try { + audioInputStream = AudioSystem.getAudioInputStream(new File(waveFile)); + } catch (Exception e) { + e.printStackTrace(); + } + format = audioInputStream.getFormat(); + double[] signal = MaryAudioUtils.getSamplesAsDoubleArray(audioInputStream); + + /* + * // get start and end pause durations from labels double sPauseDuration = getStartPauseDuration(vocalLabels); double + * ePauseDuration = getEndPauseDuration(vocalLabels); + * + * try { int startStampIndex = (int) (sPauseDuration * format.getSampleRate()); int endStampIndex = signal.length - (int) + * (ePauseDuration * format.getSampleRate()) - 1; double[] newSignal = new double[endStampIndex-startStampIndex]; + * System.arraycopy(signal, startStampIndex, newSignal, 0, endStampIndex-startStampIndex); + * MaryAudioUtils.writeWavFile(newSignal, outwaveFile, format); } catch(Exception e){ + * System.out.println("problem : "+baseName); } + */ + + // get start and end pause durations from labels + double sPauseStamp = getStartTimeStamp(vocalLabels); + double ePauseStamp = getEndTimeStamp(vocalLabels); + + int startStampIndex = (int) (sPauseStamp * format.getSampleRate()); + int endStampIndex = (int) (ePauseStamp * format.getSampleRate()); + double[] newSignal = new double[endStampIndex - startStampIndex]; + System.arraycopy(signal, startStampIndex, newSignal, 0, endStampIndex - startStampIndex); + MaryAudioUtils.writeWavFile(newSignal, outwaveFile, format); + UnitLabel[] newVocalLabels = removePausesFromLabels(vocalLabels); + UnitLabel.writeLabFile(newVocalLabels, outlabelFile); + + } + + private UnitLabel[] removePausesFromLabels(UnitLabel[] vocalLabels) { + + ArrayList uLabels = new ArrayList(); + + // put into a list + for (int i = 0; i < vocalLabels.length; i++) { + uLabels.add(vocalLabels[i]); + } + + // remove back pauses first + for (int i = (uLabels.size() - 1); i > 0; i--) { + if ("_".equals(vocalLabels[i].unitName)) { + uLabels.remove(i); + } else { + break; + } + } + + // remove front pauses next + for (int i = 0; i < uLabels.size(); i++) { + if ("_".equals(vocalLabels[i].unitName)) { + uLabels.remove(i); + } else { + break; + } + } + + // post process unit labels + double removedPauseTime = (uLabels.get(0)).getStartTime(); + for (int i = 0; i < uLabels.size(); i++) { + + UnitLabel ulab = uLabels.get(i); + uLabels.remove(i); + double sTime = ulab.getStartTime(); + double eTime = ulab.getEndTime(); + ulab.setStartTime(sTime - removedPauseTime); + ulab.setEndTime(eTime - removedPauseTime); + uLabels.add(i, ulab); + } + + return uLabels.toArray(new UnitLabel[0]); + } + + /** + * + * @param vocalLabels + * @return + */ + private double getStartPauseDuration(UnitLabel[] vocalLabels) { + + boolean isStartPause = false; + for (int i = 0; i < vocalLabels.length; i++) { + + if (i == 0 && "_".equals(vocalLabels[i].unitName)) { + isStartPause = true; + continue; + } + + if (isStartPause && !"_".equals(vocalLabels[i].unitName)) { + return vocalLabels[i].startTime; + } + } + + return 0.0; + } + + /** + * + * @param vocalLabels + * @return + */ + private double getEndPauseDuration(UnitLabel[] vocalLabels) { + + boolean isEndPause = false; + + for (int i = (vocalLabels.length - 1); i > 0; i--) { + + if (i == (vocalLabels.length - 1) && "_".equals(vocalLabels[i].unitName)) { + isEndPause = true; + continue; + } + + if (isEndPause && !"_".equals(vocalLabels[i].unitName)) { + return (vocalLabels[vocalLabels.length - 1].endTime - vocalLabels[i].endTime); + } + } + + return 0.0; + } + + /** + * + * @param vocalLabels + * @return + */ + private double getStartTimeStamp(UnitLabel[] vocalLabels) { + + boolean isStartPause = false; + for (int i = 0; i < vocalLabels.length; i++) { + + if (i == 0 && "_".equals(vocalLabels[i].unitName)) { + isStartPause = true; + continue; + } + + if (isStartPause && !"_".equals(vocalLabels[i].unitName)) { + return vocalLabels[i].startTime; + } + } + + return 0.0; + } + + /** + * + * @param vocalLabels + * @return + */ + private double getEndTimeStamp(UnitLabel[] vocalLabels) { + + boolean isEndPause = false; + + for (int i = (vocalLabels.length - 1); i > 0; i--) { + + if (i == (vocalLabels.length - 1) && "_".equals(vocalLabels[i].unitName)) { + isEndPause = true; + continue; + } + + if (isEndPause && !"_".equals(vocalLabels[i].unitName)) { + return vocalLabels[i].endTime; + } + } + + return vocalLabels[vocalLabels.length - 1].endTime; + } + + /** + * @param args + * @throws IOException + */ + public static void main(String[] args) throws IOException { + + // String inDirName = "/home/sathish/phd/data/original-lab-wav-sync"; + // String outDirName = "/home/sathish/phd/data/pauseless-lab-wav-sync"; + String inDirName = "/home/sathish/phd/data/original_stimulus_sync"; + String outDirName = "/home/sathish/phd/data/pauseless_stimulus_sync"; + BasenameList bnl = new BasenameList(inDirName, ".wav"); + + VocalisationLabelInspector vli = new VocalisationLabelInspector(inDirName, outDirName); + + for (int i = 0; i < bnl.getLength(); i++) { + vli.process(bnl.getName(i)); + } + + } + +} diff --git a/marytts-purgatory/marytts-builder/src/main/java/marytts/vocalizations/VocalizationAnnotationReader.java b/marytts-purgatory/marytts-builder/src/main/java/marytts/vocalizations/VocalizationAnnotationReader.java new file mode 100644 index 0000000000..fa5a2e847d --- /dev/null +++ b/marytts-purgatory/marytts-builder/src/main/java/marytts/vocalizations/VocalizationAnnotationReader.java @@ -0,0 +1,128 @@ +/** + * Copyright 2010 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.vocalizations; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; + +import marytts.util.io.BasenameList; + +/** + * Reader class for manual annotation of vocalizations + * + * The format of the file should be as following examples: (first line indicates list of feature names) + * + * categories|name|voicequality|angry|sadness|amusement|happiness|contempt|solidarity|antagonism|certain|agreeing|interested| + * anticipation Obadiah_049|right|modal|2|3|1|1|2|2|1|0|0|-1|0 Poppy2_078|yeah|modal|1|1|1|2|1|2|1|-1|0|-1|0 + * Prudence_058|(laughter)|breathy|1|1|3|2|1|2|1|0|0|0|-1 Poppy2_085|yeah|breathy|1|2|1|1|1|2|1|0|0|-1|0 + * + * + * @author sathish + * + */ +public class VocalizationAnnotationReader { + + private ArrayList featureCategories; // feature categories + private Map> annotationData; // basename --> (feature category, feature value) + private BasenameList bnl = null; + + public VocalizationAnnotationReader(String featureFile) throws IOException { + this(featureFile, null); + } + + public VocalizationAnnotationReader(String featureFile, BasenameList bnl) throws IOException { + this.bnl = bnl; + formatChecker(featureFile); + featureCategories = new ArrayList(); + annotationData = new HashMap>(); + getAnnotations(featureFile); + } + + /** + * + * @param featureFile + * @throws IOException + */ + private void formatChecker(String featureFile) throws IOException { + BufferedReader bfrMean = new BufferedReader(new FileReader(new File(featureFile))); + String lineMeaning = bfrMean.readLine(); + String[] mlines = lineMeaning.split("\\|"); + int noOfStrings = mlines.length; + + while ((lineMeaning = bfrMean.readLine()) != null) { + mlines = lineMeaning.split("\\|"); + if (noOfStrings != mlines.length) { + throw new RuntimeException("the format of the file is not good."); + } + } + + System.out.println("The format of the file is good"); + bfrMean.close(); + } + + private void getAnnotations(String featureFile) throws IOException { + + BufferedReader bfrMean = new BufferedReader(new FileReader(new File(featureFile))); + String lineMeaning = bfrMean.readLine(); + String[] mlines = lineMeaning.split("\\|"); + + // read feature categories + for (int i = 1; i < mlines.length; i++) { + featureCategories.add(mlines[i].trim()); + } + + while ((lineMeaning = bfrMean.readLine()) != null) { + mlines = lineMeaning.split("\\|"); + if (bnl != null) { + if (!bnl.contains(mlines[0].trim())) { + continue; + } + } + Map featureMap = new HashMap(); + // read feature categories + for (int i = 1; i < mlines.length; i++) { + featureMap.put(featureCategories.get(i - 1), mlines[i].trim()); + } + annotationData.put(mlines[0].trim(), featureMap); + } + bfrMean.close(); + } + + public Map> getVocalizationsAnnotation() { + return this.annotationData; + } + + public ArrayList getFeatureList() { + return this.featureCategories; + } + + /** + * @param args + */ + public static void main(String[] args) { + + } + +} diff --git a/marytts-purgatory/src/main/java/marytts/modules/CARTF0Modeller.java b/marytts-purgatory/marytts-runtime/src/main/java/marytts/modules/CARTF0Modeller.java similarity index 100% rename from marytts-purgatory/src/main/java/marytts/modules/CARTF0Modeller.java rename to marytts-purgatory/marytts-runtime/src/main/java/marytts/modules/CARTF0Modeller.java diff --git a/marytts-purgatory/src/main/java/marytts/unitselection/interpolation/InterpolatingSynthesizer.java b/marytts-purgatory/marytts-runtime/src/main/java/marytts/unitselection/interpolation/InterpolatingSynthesizer.java similarity index 100% rename from marytts-purgatory/src/main/java/marytts/unitselection/interpolation/InterpolatingSynthesizer.java rename to marytts-purgatory/marytts-runtime/src/main/java/marytts/unitselection/interpolation/InterpolatingSynthesizer.java diff --git a/marytts-purgatory/src/main/java/marytts/unitselection/interpolation/InterpolatingVoice.java b/marytts-purgatory/marytts-runtime/src/main/java/marytts/unitselection/interpolation/InterpolatingVoice.java similarity index 100% rename from marytts-purgatory/src/main/java/marytts/unitselection/interpolation/InterpolatingVoice.java rename to marytts-purgatory/marytts-runtime/src/main/java/marytts/unitselection/interpolation/InterpolatingVoice.java diff --git a/marytts-purgatory/src/main/java/marytts/unitselection/select/VocalizationFFRTargetCostFunction.java b/marytts-purgatory/marytts-runtime/src/main/java/marytts/unitselection/select/VocalizationFFRTargetCostFunction.java similarity index 100% rename from marytts-purgatory/src/main/java/marytts/unitselection/select/VocalizationFFRTargetCostFunction.java rename to marytts-purgatory/marytts-runtime/src/main/java/marytts/unitselection/select/VocalizationFFRTargetCostFunction.java diff --git a/marytts-purgatory/src/main/java/marytts/vocalizations/FDPSOLASynthesisTechnology.java b/marytts-purgatory/marytts-runtime/src/main/java/marytts/vocalizations/FDPSOLASynthesisTechnology.java similarity index 100% rename from marytts-purgatory/src/main/java/marytts/vocalizations/FDPSOLASynthesisTechnology.java rename to marytts-purgatory/marytts-runtime/src/main/java/marytts/vocalizations/FDPSOLASynthesisTechnology.java diff --git a/marytts-purgatory/src/main/java/marytts/vocalizations/HNMFeatureFileReader.java b/marytts-purgatory/marytts-runtime/src/main/java/marytts/vocalizations/HNMFeatureFileReader.java similarity index 100% rename from marytts-purgatory/src/main/java/marytts/vocalizations/HNMFeatureFileReader.java rename to marytts-purgatory/marytts-runtime/src/main/java/marytts/vocalizations/HNMFeatureFileReader.java diff --git a/marytts-purgatory/src/main/java/marytts/vocalizations/HNMSynthesisTechnology.java b/marytts-purgatory/marytts-runtime/src/main/java/marytts/vocalizations/HNMSynthesisTechnology.java similarity index 100% rename from marytts-purgatory/src/main/java/marytts/vocalizations/HNMSynthesisTechnology.java rename to marytts-purgatory/marytts-runtime/src/main/java/marytts/vocalizations/HNMSynthesisTechnology.java diff --git a/marytts-purgatory/src/main/java/marytts/vocalizations/KMeansClusterer.java b/marytts-purgatory/marytts-runtime/src/main/java/marytts/vocalizations/KMeansClusterer.java similarity index 100% rename from marytts-purgatory/src/main/java/marytts/vocalizations/KMeansClusterer.java rename to marytts-purgatory/marytts-runtime/src/main/java/marytts/vocalizations/KMeansClusterer.java diff --git a/marytts-purgatory/src/main/java/marytts/vocalizations/MLSAFeatureFileReader.java b/marytts-purgatory/marytts-runtime/src/main/java/marytts/vocalizations/MLSAFeatureFileReader.java similarity index 100% rename from marytts-purgatory/src/main/java/marytts/vocalizations/MLSAFeatureFileReader.java rename to marytts-purgatory/marytts-runtime/src/main/java/marytts/vocalizations/MLSAFeatureFileReader.java diff --git a/marytts-purgatory/src/main/java/marytts/vocalizations/MLSASynthesisTechnology.java b/marytts-purgatory/marytts-runtime/src/main/java/marytts/vocalizations/MLSASynthesisTechnology.java similarity index 100% rename from marytts-purgatory/src/main/java/marytts/vocalizations/MLSASynthesisTechnology.java rename to marytts-purgatory/marytts-runtime/src/main/java/marytts/vocalizations/MLSASynthesisTechnology.java diff --git a/marytts-purgatory/src/main/java/marytts/vocalizations/SourceTargetPair.java b/marytts-purgatory/marytts-runtime/src/main/java/marytts/vocalizations/SourceTargetPair.java similarity index 100% rename from marytts-purgatory/src/main/java/marytts/vocalizations/SourceTargetPair.java rename to marytts-purgatory/marytts-runtime/src/main/java/marytts/vocalizations/SourceTargetPair.java diff --git a/marytts-purgatory/src/main/java/marytts/vocalizations/VocalizationCandidate.java b/marytts-purgatory/marytts-runtime/src/main/java/marytts/vocalizations/VocalizationCandidate.java similarity index 100% rename from marytts-purgatory/src/main/java/marytts/vocalizations/VocalizationCandidate.java rename to marytts-purgatory/marytts-runtime/src/main/java/marytts/vocalizations/VocalizationCandidate.java diff --git a/marytts-purgatory/src/main/java/marytts/vocalizations/VocalizationFeatureFileReader.java b/marytts-purgatory/marytts-runtime/src/main/java/marytts/vocalizations/VocalizationFeatureFileReader.java similarity index 100% rename from marytts-purgatory/src/main/java/marytts/vocalizations/VocalizationFeatureFileReader.java rename to marytts-purgatory/marytts-runtime/src/main/java/marytts/vocalizations/VocalizationFeatureFileReader.java diff --git a/marytts-purgatory/src/main/java/marytts/vocalizations/VocalizationIntonationReader.java b/marytts-purgatory/marytts-runtime/src/main/java/marytts/vocalizations/VocalizationIntonationReader.java similarity index 100% rename from marytts-purgatory/src/main/java/marytts/vocalizations/VocalizationIntonationReader.java rename to marytts-purgatory/marytts-runtime/src/main/java/marytts/vocalizations/VocalizationIntonationReader.java diff --git a/marytts-purgatory/src/main/java/marytts/vocalizations/VocalizationSelector.java b/marytts-purgatory/marytts-runtime/src/main/java/marytts/vocalizations/VocalizationSelector.java similarity index 100% rename from marytts-purgatory/src/main/java/marytts/vocalizations/VocalizationSelector.java rename to marytts-purgatory/marytts-runtime/src/main/java/marytts/vocalizations/VocalizationSelector.java diff --git a/marytts-purgatory/src/main/java/marytts/vocalizations/VocalizationSynthesisTechnology.java b/marytts-purgatory/marytts-runtime/src/main/java/marytts/vocalizations/VocalizationSynthesisTechnology.java similarity index 100% rename from marytts-purgatory/src/main/java/marytts/vocalizations/VocalizationSynthesisTechnology.java rename to marytts-purgatory/marytts-runtime/src/main/java/marytts/vocalizations/VocalizationSynthesisTechnology.java diff --git a/marytts-purgatory/src/main/java/marytts/vocalizations/VocalizationSynthesizer.java b/marytts-purgatory/marytts-runtime/src/main/java/marytts/vocalizations/VocalizationSynthesizer.java similarity index 100% rename from marytts-purgatory/src/main/java/marytts/vocalizations/VocalizationSynthesizer.java rename to marytts-purgatory/marytts-runtime/src/main/java/marytts/vocalizations/VocalizationSynthesizer.java diff --git a/marytts-purgatory/src/main/java/marytts/vocalizations/VocalizationUnit.java b/marytts-purgatory/marytts-runtime/src/main/java/marytts/vocalizations/VocalizationUnit.java similarity index 100% rename from marytts-purgatory/src/main/java/marytts/vocalizations/VocalizationUnit.java rename to marytts-purgatory/marytts-runtime/src/main/java/marytts/vocalizations/VocalizationUnit.java diff --git a/marytts-purgatory/src/main/java/marytts/vocalizations/VocalizationUnitFileReader.java b/marytts-purgatory/marytts-runtime/src/main/java/marytts/vocalizations/VocalizationUnitFileReader.java similarity index 100% rename from marytts-purgatory/src/main/java/marytts/vocalizations/VocalizationUnitFileReader.java rename to marytts-purgatory/marytts-runtime/src/main/java/marytts/vocalizations/VocalizationUnitFileReader.java From a686bb904fe92df7f11dbdb13e9b7cb67542c042 Mon Sep 17 00:00:00 2001 From: Tristan Hamilton Date: Mon, 9 Feb 2015 17:23:16 +0100 Subject: [PATCH 27/31] add unitselection config to unitselectionmodule, remove from synthesiser list in marybase.config --- .../resources/marytts/config/marybase.config | 1 - .../unitselection/UnitSelectionConfig.java | 33 +++++++++++ .../services/marytts.config.MaryConfig | 1 + .../unitselection/config/unitselection.config | 56 +++++++++++++++++++ 4 files changed, 90 insertions(+), 1 deletion(-) create mode 100644 marytts-unitselection/src/main/java/marytts/unitselection/UnitSelectionConfig.java create mode 100644 marytts-unitselection/src/main/resources/META-INF/services/marytts.config.MaryConfig create mode 100644 marytts-unitselection/src/main/resources/marytts/unitselection/config/unitselection.config diff --git a/marytts-runtime/src/main/resources/marytts/config/marybase.config b/marytts-runtime/src/main/resources/marytts/config/marybase.config index 64e0e1ea4d..721b27543f 100644 --- a/marytts-runtime/src/main/resources/marytts/config/marybase.config +++ b/marytts-runtime/src/main/resources/marytts/config/marybase.config @@ -102,7 +102,6 @@ modules.classes.list = \ # These have to implement marytts.modules.synthesis.WaveformSynthesizer. synthesizers.classes.list = \ - marytts.unitselection.UnitSelectionSynthesizer \ marytts.modules.synthesis.HMMSynthesizer \ # Java classes to use as the audio effects post-processors, as a whitespace-separated list. diff --git a/marytts-unitselection/src/main/java/marytts/unitselection/UnitSelectionConfig.java b/marytts-unitselection/src/main/java/marytts/unitselection/UnitSelectionConfig.java new file mode 100644 index 0000000000..244ba8c3af --- /dev/null +++ b/marytts-unitselection/src/main/java/marytts/unitselection/UnitSelectionConfig.java @@ -0,0 +1,33 @@ +/** + * Copyright 2015 DFKI GmbH. + * All Rights Reserved. Use is subject to license terms. + * + * This file is part of MARY TTS. + * + * MARY TTS is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + */ +package marytts.unitselection; + +import marytts.config.SynthesisConfig; +import marytts.exceptions.MaryConfigurationException; + +public class UnitSelectionConfig extends SynthesisConfig { + public UnitSelectionConfig() throws MaryConfigurationException { + super(UnitSelectionConfig.class.getResourceAsStream("unitselection.config")); + } +} + + + + diff --git a/marytts-unitselection/src/main/resources/META-INF/services/marytts.config.MaryConfig b/marytts-unitselection/src/main/resources/META-INF/services/marytts.config.MaryConfig new file mode 100644 index 0000000000..4feea4cac2 --- /dev/null +++ b/marytts-unitselection/src/main/resources/META-INF/services/marytts.config.MaryConfig @@ -0,0 +1 @@ +marytts.unitselection.UnitSelectionConfig diff --git a/marytts-unitselection/src/main/resources/marytts/unitselection/config/unitselection.config b/marytts-unitselection/src/main/resources/marytts/unitselection/config/unitselection.config new file mode 100644 index 0000000000..f33e7be516 --- /dev/null +++ b/marytts-unitselection/src/main/resources/marytts/unitselection/config/unitselection.config @@ -0,0 +1,56 @@ +########################################################################## +# Copyright (C) 2006,2009 DFKI GmbH. +# All rights reserved. Use is subject to license terms. +# +# Permission is hereby granted, free of charge, to use and distribute +# this software and its documentation without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of this work, and to +# permit persons to whom this work is furnished to do so, subject to +# the following conditions: +# +# 1. The code must retain the above copyright notice, this list of +# conditions and the following disclaimer. +# 2. Any modifications must be clearly marked as such. +# 3. Original authors' names are not deleted. +# 4. The authors' names are not used to endorse or promote products +# derived from this software without specific prior written +# permission. +# +# DFKI GMBH AND THE CONTRIBUTORS TO THIS WORK DISCLAIM ALL WARRANTIES WITH +# REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL DFKI GMBH NOR THE +# CONTRIBUTORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL +# DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR +# PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS +# ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF +# THIS SOFTWARE. +########################################################################## +# MARY TTS configuration file. +########################################################################## + +name = unitselection +unitselection.version = 4.3.0 + +########################################################################### +############################## The Modules ############################### +########################################################################### +# For keys ending in ".list", values will be appended across config files, +# so that .list keys can occur in several config files. +# For all other keys, values will be copied to the global config, so +# keys should be unique across config files. + +# Java classes to use as the modules, as a whitespace-separated list. +# These have to implement marytts.modules.MaryModule. +# For each input type + output type, only one Module may be listed +# across all components, or an Exception will be thrown. +# No particular order is required -- a processing pipeline from input data type +# to output data type will be computed on-line. + +synthesizers.classes.list = \ + marytts.unitselection.UnitSelectionSynthesizer \ + +#################################################################### +####################### Module settings ########################### +#################################################################### + From 04298b0e26ef75a6120a14944a723c7a0af38ecf Mon Sep 17 00:00:00 2001 From: Tristan Hamilton Date: Wed, 11 Feb 2015 14:54:05 +0100 Subject: [PATCH 28/31] 'cherry picked' more files from ali's InfoRequestHandler-Code-Refactored branch I didn't realise were needed. on a side note, this is what i have not taken from his branch and thus must still be committed in: -timelineTest.bin -all files related to exceptions showing actual costs fix -> they need to be formatted as they are causing conflicts -also, although it has been added, the DocumentTest class needs to be cleaned --- marytts-client/pom.xml | 6 +- .../main/java/marytts/client/MaryClient.java | 2 +- .../java/marytts/client/MaryFormData.java | 135 ++++--- .../src/main/java/marytts/server/Request.java | 49 ++- .../server/http/FileRequestHandler.java | 5 +- .../resources/marytts/server/http/index.html | 2 + .../resources/marytts/server/http/mary.js | 22 +- .../marytts/server/http/vkbeautify.js | 358 ++++++++++++++++++ .../java/marytts/signalproc/Defaults.java | 7 +- 9 files changed, 511 insertions(+), 75 deletions(-) create mode 100644 marytts-runtime/src/main/resources/marytts/server/http/vkbeautify.js diff --git a/marytts-client/pom.xml b/marytts-client/pom.xml index 5300736d9f..238e629bce 100644 --- a/marytts-client/pom.xml +++ b/marytts-client/pom.xml @@ -33,7 +33,11 @@ junit test - + + + com.google.code.gson + gson + org.incava java-diff diff --git a/marytts-client/src/main/java/marytts/client/MaryClient.java b/marytts-client/src/main/java/marytts/client/MaryClient.java index 25dc8fbf68..a0a667bbf9 100644 --- a/marytts-client/src/main/java/marytts/client/MaryClient.java +++ b/marytts-client/src/main/java/marytts/client/MaryClient.java @@ -362,7 +362,7 @@ public Vector getOutputDataTypes() throws IOException { * if the host could not be found */ public Vector getVoices() throws IOException { - if (data.allVoices == null) + if (data.allVoices == null || data.allVoices.isEmpty()) fillVoices(); assert data.allVoices != null && data.allVoices.size() > 0; diff --git a/marytts-client/src/main/java/marytts/client/MaryFormData.java b/marytts-client/src/main/java/marytts/client/MaryFormData.java index 1f52ddd813..63bab0f151 100644 --- a/marytts-client/src/main/java/marytts/client/MaryFormData.java +++ b/marytts-client/src/main/java/marytts/client/MaryFormData.java @@ -27,8 +27,8 @@ import java.util.StringTokenizer; import java.util.Vector; -import javax.sound.sampled.AudioSystem; import javax.sound.sampled.AudioFileFormat.Type; +import javax.sound.sampled.AudioSystem; import marytts.util.MaryUtils; import marytts.util.data.audio.MaryAudioUtils; @@ -36,6 +36,11 @@ import marytts.util.math.MathUtils; import marytts.util.string.StringUtils; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; + /** * * This class nests all the information and functions that a Mary client needs to receive/send data from/to server. To be able to @@ -83,9 +88,11 @@ public class MaryFormData { public AudioEffectsBoxData effectsBoxData; public Vector limitedDomainExampleTexts; - public Map keyValuePairs; // Key-Value pairs for communication with server + public Map keyValuePairs; // Key-Value pairs for + // communication with server - public String outputAudioResponseID; // output audio file for web browser client + public String outputAudioResponseID; // output audio file for web browser + // client public String mimeType; // MIME type for output audio (web browser clients) public MaryFormData() { @@ -194,56 +201,40 @@ public void toVoices(String info) { allVoices = null; voicesByLocaleMap = null; limitedDomainVoices = null; - - if (info != null && info.length() > 0) { + JsonArray voiceArray = null; + try { + JsonParser parser = new JsonParser(); + JsonElement jsonElement = parser.parse(info); + voiceArray = jsonElement.getAsJsonArray(); + } catch (Exception e) { + e.printStackTrace(); + } + // if (info!=null && info.length()>0) + if (voiceArray != null && voiceArray.size() > 0) { allVoices = new Vector(); voicesByLocaleMap = new HashMap>(); limitedDomainVoices = new HashMap>(); - String[] voiceStrings = info.split("\n"); - - for (int i = 0; i < voiceStrings.length; i++) { - StringTokenizer st = new StringTokenizer(voiceStrings[i]); - if (!st.hasMoreTokens()) - continue; // ignore entry - String name = st.nextToken(); - if (!st.hasMoreTokens()) - continue; // ignore entry - String localeString = st.nextToken(); + // String[] voiceStrings = info.split("\n"); + for (int i = 0; i < voiceArray.size(); i++) { + JsonObject currentVoice = (JsonObject) voiceArray.get(i); + String name = currentVoice.has("name") ? currentVoice.get("name").toString().replace("\"", "") : ""; + String localeString = currentVoice.has("locale") ? currentVoice.get("locale").toString().replace("\"", "") : ""; Locale locale = string2locale(localeString); + // locale = string2locale("en-US"); assert locale != null; - if (!st.hasMoreTokens()) - continue; // ignore entry - String gender = st.nextToken(); - + String gender = currentVoice.has("gender") ? currentVoice.get("gender").toString().replace("\"", "") : ""; MaryClient.Voice voice = null; + if (!currentVoice.has("domain")) { + voice = new MaryClient.Voice(name, locale, gender, "general"); + } else { + String domain = currentVoice.get("domain").toString().replace("\"", ""); + voice = new MaryClient.Voice(name, locale, gender, domain); + } if (isServerVersionAtLeast("3.5.0")) { - String synthesizerType; - if (!st.hasMoreTokens()) - synthesizerType = "non-specified"; - else - synthesizerType = st.nextToken(); - - if (!st.hasMoreTokens()) { - // assume domain is general - voice = new MaryClient.Voice(name, locale, gender, "general"); - } else { - // read in the domain - String domain = st.nextToken(); - voice = new MaryClient.Voice(name, locale, gender, domain); - } - + String synthesizerType = currentVoice.has("type") ? currentVoice.get("type").toString().replace("\"", "") + : "non-specified"; voice.setSynthesizerType(synthesizerType); - } else { - if (!st.hasMoreTokens()) { - // assume domain is general - voice = new MaryClient.Voice(name, locale, gender, "general"); - } else { - // read in the domain - String domain = st.nextToken(); - voice = new MaryClient.Voice(name, locale, gender, domain); - } } - allVoices.add(voice); Vector localeVoices = null; if (voicesByLocaleMap.containsKey(locale)) { @@ -254,6 +245,56 @@ public void toVoices(String info) { } localeVoices.add(voice); } + // for (int i=0; i localeVoices = null; + // if (voicesByLocaleMap.containsKey(locale)) { + // localeVoices = voicesByLocaleMap.get(locale); + // } else { + // localeVoices = new Vector(); + // voicesByLocaleMap.put(locale, localeVoices); + // } + // localeVoices.add(voice); + // } } } @@ -517,7 +558,8 @@ else if (serverExampleTexts != null) // Check box selected = keyValuePairs.get("effect_" + currentEffectName + "_selected"); - if (selected != null && selected.compareTo("on") == 0) // Effect is selected + if (selected != null && selected.compareTo("on") == 0) // Effect is + // selected effectsBoxData.getControlData(i).setSelected(true); else // If not found, effect is not selected @@ -544,7 +586,8 @@ public boolean isServerVersionAtLeast(String serverVersionToCompare) { return tmp >= 0; } - // Check if all selections are appropriately made, i.e. no array bounds exceeded etc + // Check if all selections are appropriately made, i.e. no array bounds + // exceeded etc public void checkAndCorrectSelections() { audioFormatSelected = MathUtils.CheckLimits(audioFormatSelected, 0, audioFileFormatTypes.size() - 1); audioOutSelected = MathUtils.CheckLimits(audioOutSelected, 0, audioOutTypes.size() - 1); diff --git a/marytts-runtime/src/main/java/marytts/server/Request.java b/marytts-runtime/src/main/java/marytts/server/Request.java index e0f09f2a23..f130f272e2 100644 --- a/marytts-runtime/src/main/java/marytts/server/Request.java +++ b/marytts-runtime/src/main/java/marytts/server/Request.java @@ -48,6 +48,7 @@ import marytts.modules.MaryModule; import marytts.modules.ModuleRegistry; import marytts.modules.synthesis.Voice; +import marytts.signalproc.Defaults; import marytts.util.MaryCache; import marytts.util.MaryRuntimeUtils; import marytts.util.MaryUtils; @@ -78,6 +79,8 @@ * data is either accessed directly (getOutputData()) or written to an output stream (writeOutputData). */ public class Request { + private static final String FILENAME = "Request.java:"; + protected MaryDataType inputType; protected MaryDataType outputType; protected String outputTypeParams; @@ -109,9 +112,9 @@ public Request(MaryDataType inputType, MaryDataType outputType, Locale defaultLo String defaultEffects, String defaultStyle, int id, AudioFileFormat audioFileFormat, boolean streamAudio, String outputTypeParams) { if (!inputType.isInputType()) - throw new IllegalArgumentException("not an input type: " + inputType.name()); + throw new IllegalArgumentException(FILENAME + " not an input type: " + inputType.name()); if (!outputType.isOutputType()) - throw new IllegalArgumentException("not an output type: " + outputType.name()); + throw new IllegalArgumentException(FILENAME + " not an output type: " + outputType.name()); this.inputType = inputType; this.outputType = outputType; this.defaultLocale = defaultLocale; @@ -123,7 +126,7 @@ public Request(MaryDataType inputType, MaryDataType outputType, Locale defaultLo this.streamAudio = streamAudio; if (outputType == MaryDataType.get("AUDIO")) { if (audioFileFormat == null) - throw new NullPointerException("audio file format is needed for output type AUDIO"); + throw new NullPointerException(FILENAME + " audio file format is needed for output type AUDIO"); this.appendableAudioStream = new AppendableSequenceAudioInputStream(audioFileFormat.getFormat(), null); } else { this.appendableAudioStream = null; @@ -262,11 +265,12 @@ public void process() throws Exception { assert Mary.currentState() == Mary.STATE_RUNNING; long startTime = System.currentTimeMillis(); if (inputData == null) - throw new NullPointerException("Input data is not set."); + throw new NullPointerException(FILENAME + " Input data is not set."); if (inputType.isXMLType() && inputData.getDocument() == null) - throw new NullPointerException("Input data contains no XML document."); + throw new NullPointerException(FILENAME + " Input data contains no XML document."); if (inputType.isMaryXML() && !inputData.getDocument().getDocumentElement().hasAttribute("xml:lang")) - throw new IllegalArgumentException("Mandatory attribute xml:lang is missing from maryxml document element."); + throw new IllegalArgumentException(FILENAME + + " Mandatory attribute xml:lang is missing from maryxml document element."); NodeList inputDataList; MaryData rawmaryxml; @@ -501,9 +505,12 @@ private MaryData processOneChunk(MaryData oneInputData, MaryDataType oneOutputTy if (neededModules == null) { // The modules we have cannot be combined such that // the outputType can be generated from the inputData type. - String message = "No known way of generating output (" + oneOutputType.name() + ") from input(" - + oneInputData.getType().name() + "), no processing path through modules."; - throw new UnsupportedOperationException(message); + String inputType = oneInputData.getType().toString(); + String outputType = oneOutputType.toString(); + String tempLocale = locale.toString(); + String message = "input type: " + inputType + "\noutput type: " + outputType + "\nlocale: " + tempLocale + + "\nNo known way of generating output from input -- " + "no processing path through modules."; + throw new UnsupportedOperationException(FILENAME + " " + message); } usedModules.addAll(neededModules); logger.info("Handling request using the following modules:"); @@ -547,11 +554,12 @@ private MaryData processOneChunk(MaryData oneInputData, MaryDataType oneOutputTy try { outData = m.process(currentData); } catch (Exception e) { - throw new Exception("Module " + m.name() + ": Problem processing the data.", e); + throw new Exception(FILENAME + " Module " + m.name() + ": Problem processing the data." + "\tCause: " + + (e.getCause() != null ? e.getCause().getMessage() : "Cause is null!"), e); } if (outData == null) { - throw new NullPointerException("Module " + m.name() + " returned null. This should not happen."); + throw new NullPointerException(FILENAME + " Module " + m.name() + " returned null. This should not happen."); } outData.setDefaultVoice(defaultVoice); outData.setDefaultStyle(defaultStyle); @@ -593,9 +601,9 @@ private NodeList splitIntoChunks(MaryData rawmaryxml) { // TODO: replace this with code that combines this and the splitting functionality of Synthesis: one chunk is one locale // and one voice if (rawmaryxml == null) - throw new NullPointerException("Received null data"); + throw new NullPointerException(FILENAME + " Received null data"); if (rawmaryxml.getType() != MaryDataType.get("RAWMARYXML")) - throw new IllegalArgumentException("Expected data of type RAWMARYXML, got " + rawmaryxml.getType()); + throw new IllegalArgumentException(FILENAME + " Expected data of type RAWMARYXML, got " + rawmaryxml.getType()); if (logger.getEffectiveLevel().equals(Level.DEBUG)) { logger.debug("Now splitting the following RAWMARYXML data into chunks:"); ByteArrayOutputStream dummy = new ByteArrayOutputStream(); @@ -689,7 +697,7 @@ private NodeList splitIntoChunks(MaryData rawmaryxml) { */ private static void moveBoundariesIntoParagraphs(Document rawmaryxml) { if (rawmaryxml == null) { - throw new NullPointerException("Received null rawmaryxml"); + throw new NullPointerException(FILENAME + " Received null rawmaryxml"); } TreeWalker paraTW = ((DocumentTraversal) rawmaryxml).createTreeWalker(rawmaryxml.getDocumentElement(), NodeFilter.SHOW_ELEMENT, new NameNodeFilter(MaryXML.PARAGRAPH), true); @@ -698,7 +706,7 @@ private static void moveBoundariesIntoParagraphs(Document rawmaryxml) { // Find first paragraph, to see if there are any leading boundaries Element firstParagraph = (Element) paraTW.nextNode(); if (firstParagraph == null) { - throw new NullPointerException("Document does not have a paragraph"); + throw new NullPointerException(FILENAME + " Document does not have a paragraph"); } tw.setCurrentNode(firstParagraph); // Now move any leading boundaries to the left of into the first paragraph @@ -731,7 +739,7 @@ private static void moveBoundariesIntoParagraphs(Document rawmaryxml) { */ private static MaryData extractParagraphAsMaryData(MaryData maryxml, Element paragraph) { if (!maryxml.getType().isMaryXML()) { - throw new IllegalArgumentException("Expected MaryXML data"); + throw new IllegalArgumentException(FILENAME + " Expected MaryXML data"); } String rootLanguage = maryxml.getDocument().getDocumentElement().getAttribute("xml:lang"); Document newDoc = MaryXML.newDocument(); @@ -787,6 +795,10 @@ private Locale determineLocale(MaryData data) { if (docEl != null) { String langCode = docEl.getAttribute("xml:lang"); if (!langCode.equals("")) { + // check if locale is "en", set it to "en-US" + if (langCode.equals("en")) { + langCode = Defaults.getDefaultEnglishLocale(); + } locale = MaryUtils.string2locale(langCode); } } @@ -808,10 +820,10 @@ public MaryData getOutputData() { */ public void writeOutputData(OutputStream outputStream) throws Exception { if (outputData == null) { - throw new NullPointerException("No output data -- did process() succeed?"); + throw new NullPointerException(FILENAME + " No output data -- did process() succeed?"); } if (outputStream == null) - throw new NullPointerException("cannot write to null output stream"); + throw new NullPointerException(FILENAME + " cannot write to null output stream"); // Safety net: if the output is not written within a certain amount of // time, give up. This prevents our thread from being locked forever if an // output deadlock occurs (happened very rarely on Java 1.4.2beta). @@ -842,5 +854,4 @@ public void run() { } timer.cancel(); } - } diff --git a/marytts-runtime/src/main/java/marytts/server/http/FileRequestHandler.java b/marytts-runtime/src/main/java/marytts/server/http/FileRequestHandler.java index 46311a2212..4d9c2eee9a 100644 --- a/marytts-runtime/src/main/java/marytts/server/http/FileRequestHandler.java +++ b/marytts-runtime/src/main/java/marytts/server/http/FileRequestHandler.java @@ -40,7 +40,7 @@ public class FileRequestHandler extends BaseHttpRequestHandler { private Set validFiles = new HashSet(Arrays.asList(new String[] { "favicon.ico", "index.html", - "documentation.html", "mary.js" })); + "documentation.html", "mary.js", "vkbeautify.js" })); public FileRequestHandler() { super(); @@ -99,5 +99,4 @@ else if (resourceFilename.endsWith(".swf")) MaryHttpServerUtils.errorFileNotFound(response, resourceFilename); } } - -} +} \ No newline at end of file diff --git a/marytts-runtime/src/main/resources/marytts/server/http/index.html b/marytts-runtime/src/main/resources/marytts/server/http/index.html index 03b8e4e4a8..911ccd880f 100644 --- a/marytts-runtime/src/main/resources/marytts/server/http/index.html +++ b/marytts-runtime/src/main/resources/marytts/server/http/index.html @@ -2,6 +2,8 @@ MARY Web Client + diff --git a/marytts-runtime/src/main/resources/marytts/server/http/mary.js b/marytts-runtime/src/main/resources/marytts/server/http/mary.js index 01f4d1ef80..2f8d869061 100644 --- a/marytts-runtime/src/main/resources/marytts/server/http/mary.js +++ b/marytts-runtime/src/main/resources/marytts/server/http/mary.js @@ -66,10 +66,11 @@ function fillVoices() if (xmlHttp.readyState==4) { if (xmlHttp.status == 200) { var response = xmlHttp.responseText; + var jsonResponseArray = JSON.parse(xmlHttp.responseText); var lines = response.split('\n'); var localeElt = document.getElementById('LOCALE'); var voiceElt = document.getElementById('VOICE'); - for (l in lines) { + /*for (l in lines) { var line = lines[l]; if (line.length > 0) { addOption("VOICE_SELECTIONS", line); @@ -81,6 +82,21 @@ function fillVoices() setModificationVisibility(null, "AUDIO"); // AUDIO is default on load } } + }*/ + + for (index in jsonResponseArray) { + var jsonObject = jsonResponseArray[index]; + if (jsonObject != null) { + //format string that will be shown in the dropdown + var outputString = jsonObject.locale + " " + jsonObject.gender + " " + jsonObject.name + " " + jsonObject.type + "\r"; + addOption("VOICE_SELECTIONS", outputString); + if (localeElt.value == 'fill-me') { + voiceElt.value = jsonObject.name; + localeElt.value = jsonObject.locale; + updateInputText(true); + setModificationVisibility(null, "AUDIO"); // AUDIO is default on load + } + } } } else { alert(xmlHttp.responseText); @@ -333,7 +349,7 @@ function updateInputText(replaceInput) if (haveVoiceExample) { exampleChanged(); } else { - document.getElementById('INPUT_TEXT').value = typeExample; + document.getElementById('INPUT_TEXT').value = vkbeautify.xml(typeExample); } } } @@ -637,7 +653,7 @@ function requestSynthesis() xmlHttp.onreadystatechange = function() { if (xmlHttp.readyState==4) { if (xmlHttp.status == 200) { - document.getElementById('OUTPUT_TEXT').value = xmlHttp.responseText; + document.getElementById('OUTPUT_TEXT').value = vkbeautify.xml(xmlHttp.responseText); } else { alert(xmlHttp.responseText); } diff --git a/marytts-runtime/src/main/resources/marytts/server/http/vkbeautify.js b/marytts-runtime/src/main/resources/marytts/server/http/vkbeautify.js new file mode 100644 index 0000000000..13cee3a5ea --- /dev/null +++ b/marytts-runtime/src/main/resources/marytts/server/http/vkbeautify.js @@ -0,0 +1,358 @@ +/** +* vkBeautify - javascript plugin to pretty-print or minify text in XML, JSON, CSS and SQL formats. +* +* Version - 0.99.00.beta +* Copyright (c) 2012 Vadim Kiryukhin +* vkiryukhin @ gmail.com +* http://www.eslinstructor.net/vkbeautify/ +* +* Dual licensed under the MIT and GPL licenses: +* http://www.opensource.org/licenses/mit-license.php +* http://www.gnu.org/licenses/gpl.html +* +* Pretty print +* +* vkbeautify.xml(text [,indent_pattern]); +* vkbeautify.json(text [,indent_pattern]); +* vkbeautify.css(text [,indent_pattern]); +* vkbeautify.sql(text [,indent_pattern]); +* +* @text - String; text to beatufy; +* @indent_pattern - Integer | String; +* Integer: number of white spaces; +* String: character string to visualize indentation ( can also be a set of white spaces ) +* Minify +* +* vkbeautify.xmlmin(text [,preserve_comments]); +* vkbeautify.jsonmin(text); +* vkbeautify.cssmin(text [,preserve_comments]); +* vkbeautify.sqlmin(text); +* +* @text - String; text to minify; +* @preserve_comments - Bool; [optional]; +* Set this flag to true to prevent removing comments from @text ( minxml and mincss functions only. ) +* +* Examples: +* vkbeautify.xml(text); // pretty print XML +* vkbeautify.json(text, 4 ); // pretty print JSON +* vkbeautify.css(text, '. . . .'); // pretty print CSS +* vkbeautify.sql(text, '----'); // pretty print SQL +* +* vkbeautify.xmlmin(text, true);// minify XML, preserve comments +* vkbeautify.jsonmin(text);// minify JSON +* vkbeautify.cssmin(text);// minify CSS, remove comments ( default ) +* vkbeautify.sqlmin(text);// minify SQL +* +*/ + +(function() { + +function createShiftArr(step) { + + var space = ' '; + + if ( isNaN(parseInt(step)) ) { // argument is string + space = step; + } else { // argument is integer + switch(step) { + case 1: space = ' '; break; + case 2: space = ' '; break; + case 3: space = ' '; break; + case 4: space = ' '; break; + case 5: space = ' '; break; + case 6: space = ' '; break; + case 7: space = ' '; break; + case 8: space = ' '; break; + case 9: space = ' '; break; + case 10: space = ' '; break; + case 11: space = ' '; break; + case 12: space = ' '; break; + } + } + + var shift = ['\n']; // array of shifts + for(ix=0;ix<100;ix++){ + shift.push(shift[ix]+space); + } + return shift; +} + +function vkbeautify(){ + this.step = ' '; // 4 spaces + this.shift = createShiftArr(this.step); +}; + +vkbeautify.prototype.xml = function(text,step) { + + var ar = text.replace(/>\s{0,}<") + .replace(/ or -1) { + str += shift[deep]+ar[ix]; + inComment = true; + // end comment or // + if(ar[ix].search(/-->/) > -1 || ar[ix].search(/\]>/) > -1 || ar[ix].search(/!DOCTYPE/) > -1 ) { + inComment = false; + } + } else + // end comment or // + if(ar[ix].search(/-->/) > -1 || ar[ix].search(/\]>/) > -1) { + str += ar[ix]; + inComment = false; + } else + // // + if( /^<\w/.exec(ar[ix-1]) && /^<\/\w/.exec(ar[ix]) && + /^<[\w:\-\.\,]+/.exec(ar[ix-1]) == /^<\/[\w:\-\.\,]+/.exec(ar[ix])[0].replace('/','')) { + str += ar[ix]; + if(!inComment) deep--; + } else + // // + if(ar[ix].search(/<\w/) > -1 && ar[ix].search(/<\//) == -1 && ar[ix].search(/\/>/) == -1 ) { + str = !inComment ? str += shift[deep++]+ar[ix] : str += ar[ix]; + } else + // ... // + if(ar[ix].search(/<\w/) > -1 && ar[ix].search(/<\//) > -1) { + str = !inComment ? str += shift[deep]+ar[ix] : str += ar[ix]; + } else + // // + if(ar[ix].search(/<\//) > -1) { + str = !inComment ? str += shift[--deep]+ar[ix] : str += ar[ix]; + } else + // // + if(ar[ix].search(/\/>/) > -1 ) { + str = !inComment ? str += shift[deep]+ar[ix] : str += ar[ix]; + } else + // // + if(ar[ix].search(/<\?/) > -1) { + str += shift[deep]+ar[ix]; + } else + // xmlns // + if( ar[ix].search(/xmlns\:/) > -1 || ar[ix].search(/xmlns\=/) > -1) { + str += shift[deep]+ar[ix]; + } + + else { + str += ar[ix]; + } + } + + return (str[0] == '\n') ? str.slice(1) : str; +} + +vkbeautify.prototype.json = function(text,step) { + + var step = step ? step : this.step; + + if (typeof JSON === 'undefined' ) return text; + + if ( typeof text === "string" ) return JSON.stringify(JSON.parse(text), null, step); + if ( typeof text === "object" ) return JSON.stringify(text, null, step); + + return text; // text is not string nor object +} + +vkbeautify.prototype.css = function(text, step) { + + var ar = text.replace(/\s{1,}/g,' ') + .replace(/\{/g,"{~::~") + .replace(/\}/g,"~::~}~::~") + .replace(/\;/g,";~::~") + .replace(/\/\*/g,"~::~/*") + .replace(/\*\//g,"*/~::~") + .replace(/~::~\s{0,}~::~/g,"~::~") + .split('~::~'), + len = ar.length, + deep = 0, + str = '', + ix = 0, + shift = step ? createShiftArr(step) : this.shift; + + for(ix=0;ix/g,"") + .replace(/[ \r\n\t]{1,}xmlns/g, ' xmlns'); + return str.replace(/>\s{0,}<"); +} + +vkbeautify.prototype.jsonmin = function(text) { + + if (typeof JSON === 'undefined' ) return text; + + return JSON.stringify(JSON.parse(text), null, 0); + +} + +vkbeautify.prototype.cssmin = function(text, preserveComments) { + + var str = preserveComments ? text + : text.replace(/\/\*([^*]|[\r\n]|(\*+([^*/]|[\r\n])))*\*+\//g,"") ; + + return str.replace(/\s{1,}/g,' ') + .replace(/\{\s{1,}/g,"{") + .replace(/\}\s{1,}/g,"}") + .replace(/\;\s{1,}/g,";") + .replace(/\/\*\s{1,}/g,"/*") + .replace(/\*\/\s{1,}/g,"*/"); +} + +vkbeautify.prototype.sqlmin = function(text) { + return text.replace(/\s{1,}/g," ").replace(/\s{1,}\(/,"(").replace(/\s{1,}\)/,")"); +} + +window.vkbeautify = new vkbeautify(); + +})(); + diff --git a/marytts-signalproc/src/main/java/marytts/signalproc/Defaults.java b/marytts-signalproc/src/main/java/marytts/signalproc/Defaults.java index 5ac3855e3d..53c72d2df1 100644 --- a/marytts-signalproc/src/main/java/marytts/signalproc/Defaults.java +++ b/marytts-signalproc/src/main/java/marytts/signalproc/Defaults.java @@ -24,9 +24,9 @@ /** * * @author Marc Schröder - * + * * A set of static getters for System properties. - * + * */ public class Defaults { public static int getWindowSize() { @@ -48,4 +48,7 @@ public static int getFrameShift() { return shift; } + public static String getDefaultEnglishLocale() { + return "en-US"; + } } From 99a8b1dffb8fd42af6214c61dd0fdf472870f8c1 Mon Sep 17 00:00:00 2001 From: Tristan Hamilton Date: Wed, 11 Feb 2015 15:35:59 +0100 Subject: [PATCH 29/31] needed to change unitselection.config file location so it could be read --- .../marytts/unitselection/{config => }/unitselection.config | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename marytts-unitselection/src/main/resources/marytts/unitselection/{config => }/unitselection.config (100%) diff --git a/marytts-unitselection/src/main/resources/marytts/unitselection/config/unitselection.config b/marytts-unitselection/src/main/resources/marytts/unitselection/unitselection.config similarity index 100% rename from marytts-unitselection/src/main/resources/marytts/unitselection/config/unitselection.config rename to marytts-unitselection/src/main/resources/marytts/unitselection/unitselection.config From 7230f50283e3a2cca68b53cda1c913f48e9031f4 Mon Sep 17 00:00:00 2001 From: Tristan Hamilton Date: Tue, 10 Mar 2015 14:42:03 +0100 Subject: [PATCH 30/31] created UnitSelectionIT class, required Hamcrest for testing -> updated Hamcrest dependencies to Hamcrest-integration. Hamcrest developers advise against using Hamcrest-all. The maven dependency order for using hamcrest and junit also seem to rely on hamcrest coming BEFORE junit. Also looked at implementing a mocked voice for the integration test using mockito, however it seems a real voice is needed to test. --- marytts-builder/pom.xml | 12 ++-- marytts-unitselection/pom.xml | 21 ++++++ .../unitselection/UnitSelectionIT.java | 69 +++++++++++++++++++ pom.xml | 22 ++++-- 4 files changed, 111 insertions(+), 13 deletions(-) create mode 100644 marytts-unitselection/src/test/java/marytts/unitselection/UnitSelectionIT.java diff --git a/marytts-builder/pom.xml b/marytts-builder/pom.xml index 3c4b8727a4..7f5a665b4c 100644 --- a/marytts-builder/pom.xml +++ b/marytts-builder/pom.xml @@ -47,6 +47,12 @@ commons-lang commons-lang + + + org.hamcrest + hamcrest-integration + test + junit @@ -92,12 +98,6 @@ test - - org.hamcrest - hamcrest-all - test - - org.swinglabs swing-layout diff --git a/marytts-unitselection/pom.xml b/marytts-unitselection/pom.xml index 0f5b586a6f..74d09f62c8 100644 --- a/marytts-unitselection/pom.xml +++ b/marytts-unitselection/pom.xml @@ -20,5 +20,26 @@ net.sf.trove4j trove4j + + + com.google.guava + guava + test + + + org.hamcrest + hamcrest-integration + test + + + junit + junit + test + + + org.mockito + mockito-core + test + \ No newline at end of file diff --git a/marytts-unitselection/src/test/java/marytts/unitselection/UnitSelectionIT.java b/marytts-unitselection/src/test/java/marytts/unitselection/UnitSelectionIT.java new file mode 100644 index 0000000000..a73720f984 --- /dev/null +++ b/marytts-unitselection/src/test/java/marytts/unitselection/UnitSelectionIT.java @@ -0,0 +1,69 @@ +package marytts.unitselection; + + +import java.io.IOException; +import java.util.List; +import java.util.Locale; + +import javax.sound.sampled.AudioInputStream; + +import marytts.LocalMaryInterface; +import marytts.MaryInterface; +import marytts.modules.synthesis.Voice; +import marytts.config.VoiceConfig; +import marytts.util.MaryRuntimeUtils; +import marytts.util.dom.DomUtils; +import marytts.config.MaryConfig; + +import static org.hamcrest.CoreMatchers.*; +import static org.junit.Assert.assertThat; +import org.junit.Before; +import org.junit.Test; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +import org.mockito.Mockito; + +public class UnitSelectionIT { + + MaryInterface mary; + + @Before + public void setUp() throws Exception { + mary = new LocalMaryInterface(); + } + + @Test + public void canReadUnitSelectionConfig() throws Exception { + assertThat(marytts.config.MaryConfig.getConfigs(), hasItem(isA(UnitSelectionConfig.class))); + } + + /** + @Test + public void canLoadVoice() throws Exception { + Voice voice = new UnitSelectionVoice(mockedVoiceConfig.getName(), null); + assertNotNull(voice); + } + **/ + + /** + @Test + public void canSetVoice() throws Exception { + UnitSelectionVoice mockedVoice = Mockito.mock(UnitSelectionVoice.class); + when(mockedVoice.getName()).thenReturn("fnord"); + + mary.setVoice(mockedVoice.getName()); + assertEquals("fnord", mary.getVoice()); + + } + **/ + + /** + @Test + public void canProcessTextToSpeech() throws Exception { + mary.setVoice(mockedVoiceConfig.getName()); + AudioInputStream audio = mary.generateAudio("Hello world"); + assertNotNull(audio); + } + **/ +} diff --git a/pom.xml b/pom.xml index d224bd9422..3734930065 100644 --- a/pom.xml +++ b/pom.xml @@ -205,6 +205,12 @@ 1.1 + + org.hamcrest + hamcrest-integration + 1.3 + + junit junit @@ -320,13 +326,8 @@ fest-assert 1.4 - - - org.hamcrest - hamcrest-all - 1.3 - - + + org.hsqldb hsqldb @@ -338,6 +339,13 @@ java-diff 1.1 + + + org.mockito + mockito-core + 1.10.19 + + org.swinglabs From 9b5f4cd6449c281557a2035b9d25ecdf67be1edb Mon Sep 17 00:00:00 2001 From: Tristan Hamilton Date: Tue, 10 Mar 2015 14:51:48 +0100 Subject: [PATCH 31/31] added methods to improve SynthesisConfig class and to retrieve Synthesis configs in MaryConfig. NOTE: MaryConfig.checkConsistency() is never used. --- .../main/java/marytts/config/MaryConfig.java | 20 +++++++++++++++++++ .../java/marytts/config/SynthesisConfig.java | 12 +++++++++++ 2 files changed, 32 insertions(+) diff --git a/marytts-runtime/src/main/java/marytts/config/MaryConfig.java b/marytts-runtime/src/main/java/marytts/config/MaryConfig.java index 0fd5e94905..66c829c86f 100644 --- a/marytts-runtime/src/main/java/marytts/config/MaryConfig.java +++ b/marytts-runtime/src/main/java/marytts/config/MaryConfig.java @@ -71,6 +71,15 @@ public static void checkConsistency() throws MaryConfigurationException { + "', but there is no corresponding language config."); } } + // Check that for each synthesis config, we have a matching synthesis class + if (getSynthesisConfigs() == null){ + throw new MaryConfigurationException("No synthesizer config"); + } + for (SynthesisConfig sc : getSynthesisConfigs()) { + if (MaryProperties.synthesizerClasses().contains(sc.getSynthesisName())) { + throw new MaryConfigurationException("Synthesizer '" + sc.getSynthesisName() + "' is not in config properties."); + } + } } public static int countConfigs() { @@ -145,6 +154,17 @@ public static LanguageConfig getLanguageConfig(Locale locale) { return null; } + public static Iterable getSynthesisConfigs() { + Set scs = new HashSet(); + for (MaryConfig mc : configLoader) { + if (mc.isSynthesisConfig()) { + SynthesisConfig sc = (SynthesisConfig) mc; + scs.add(sc); + } + } + return scs; + } + /** * Get the voice config for the given voice name, or null if there is no such voice config. * diff --git a/marytts-runtime/src/main/java/marytts/config/SynthesisConfig.java b/marytts-runtime/src/main/java/marytts/config/SynthesisConfig.java index 53cdd20aea..403db0f54b 100644 --- a/marytts-runtime/src/main/java/marytts/config/SynthesisConfig.java +++ b/marytts-runtime/src/main/java/marytts/config/SynthesisConfig.java @@ -33,10 +33,22 @@ public class SynthesisConfig extends MaryConfig { public SynthesisConfig(InputStream propertyStream) throws MaryConfigurationException { super(propertyStream); + if (getSynthesisName() == null) { + throw new MaryConfigurationException("No synthesizer class defined in config file"); + } } @Override public boolean isSynthesisConfig() { return true; } + + /** + * The synthesizer's name. + * + * @return + */ + public String getSynthesisName() { + return getProperties().getProperty("synthesizers.classes.list"); + } }