qtau/plugins/ecantorix_synth/ecantorix_synth.cpp

/*
    This file is part of QTau
    Copyright (C) 2013-2018  Tobias "Tomoko" Platen <tplaten@posteo.de>
    Copyright (C) 2013       digited       <https://github.com/digited>
    Copyright (C) 2010-2013  HAL@ShurabaP  <https://github.com/haruneko>

    QTau is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.

    SPDX-License-Identifier: GPL-3.0+
*/

#include "ecantorix_synth.h"
#include "../editor/ustjkeys.h"

#include <QFileInfo>
#include <QDebug>
#include <QDir>
#include <QFile>
#include <QTextStream>
#include <QDebug>
#include <QDirIterator>
#include <QStringList>
#include <QJsonDocument>

#include <math.h>
#include <stdlib.h>
#include <stdio.h>
#include <QProcess>
#include <QtConcurrent/QtConcurrent>
#include <sndfile.h>
#include <unistd.h>
#include <sekai/midi.h>
#include <assert.h>

#define __devloglevel__ 4

//utilityfunctions -- move to libkawaii
bool fileExists(QString path) {
    QFileInfo check_file(path);
    // check if file exists and if yes: Is it really a file and no directory?
    return check_file.exists() && check_file.isFile();
}

//ecantorix synth specific

/// manifest
QString eCantorixSynth::name()        { return "eCantorix2"; }
QString eCantorixSynth::description() { return "A multilingual KTH-style singing synthesizer"; }
QString eCantorixSynth::version()     { return "18.04"; }


/// setup
void eCantorixSynth::setup(IController* ctrl)      {
    this->_ctrl = ctrl;
    this->_jack_samplerate = ctrl->sampleRate();
    _wavtool.setOutputRate(this->_jack_samplerate);

    //thread queue signaling
    connect(this,&eCantorixSynth::logDebug,this,&eCantorixSynth::on_logDebug);
    connect(this,&eCantorixSynth::logError,this,&eCantorixSynth::on_logError);
    connect(this,&eCantorixSynth::logSuccess,this,&eCantorixSynth::on_logSuccess);

   // _voices["Mikulas"]="cs";
    _voices["Kurogane"]="en1";
    _voices["Anna"]="de1";
   // _voices["Lukas"]="sv";
   // _voices["Mika"]="fi";

    _phoTypes["a"]={0,VOWEL};
    _phoTypes["A"]={0,VOWEL};
    _phoTypes["{"]={0,VOWEL};
    _phoTypes["6"]={0,VOWEL};
    _phoTypes["q"]={0,VOWEL};
    _phoTypes["Q"]={0,VOWEL};
    _phoTypes["e"]={0,VOWEL};
    _phoTypes["E"]={0,VOWEL};
    _phoTypes["@"]={0,VOWEL};
    _phoTypes["3"]={0,VOWEL};
    _phoTypes["i"]={0,VOWEL};
    _phoTypes["I"]={0,VOWEL};
    _phoTypes["o"]={0,VOWEL};
    _phoTypes["O"]={0,VOWEL};
    _phoTypes["O:"]={0,VOWEL};
    _phoTypes["2"]={0,VOWEL};
    _phoTypes["9"]={0,VOWEL};
    _phoTypes["&"]={0,VOWEL};
    _phoTypes["u"]={0,VOWEL};
    _phoTypes["U"]={0,VOWEL};
    _phoTypes["}"]={0,VOWEL};
    _phoTypes["v"]={0,VOWEL};
    _phoTypes["V"]={0,VOWEL};
    _phoTypes["y"]={0,VOWEL};
    _phoTypes["Y"]={0,VOWEL};

    _phoTypes["p"]={100,HCONSONANT};
    _phoTypes["b"]={80,VCONSONANT};
    _phoTypes["t"]={100,HCONSONANT};
    _phoTypes["d"]={80,VCONSONANT};
    _phoTypes["k"]={100,HCONSONANT};
    _phoTypes["g"]={80,VCONSONANT};
    _phoTypes["pf"]={160,HCONSONANT};
    _phoTypes["ts"]={160,HCONSONANT};
    _phoTypes["tS"]={160,HCONSONANT};
    _phoTypes["f"]={50,HCONSONANT};
    _phoTypes["v"]={50,VCONSONANT};
    _phoTypes["s"]={60,HCONSONANT};
    _phoTypes["z"]={60,VCONSONANT};
    _phoTypes["S"]={60,HCONSONANT};
    _phoTypes["Z"]={60,VCONSONANT};
    _phoTypes["C"]={60,HCONSONANT};
    _phoTypes["j"]={60,VCONSONANT};
    _phoTypes["x"]={700,VCONSONANT};
    _phoTypes["h"]={100,HCONSONANT};
    _phoTypes["m"]={60,NASAL};
    _phoTypes["n"]={60,NASAL};
    _phoTypes["N"]={60,NASAL};
    _phoTypes["l"]={60,LIQUID};
    _phoTypes["R"]={60,LIQUID};



}

bool eCantorixSynth::setCacheDir(QString cacheDir)
{
    //FIX this
    DEVLOG_DEBUG("cacheDir: "+cacheDir);
    return true;
}

bool eCantorixSynth::synthIsRealtime()
{
    return false;
}

void eCantorixSynth::lookupPho(QString pho)
{
phoEvent e;
   e.type = _phoTypes[pho];
   e.symbol = pho;
   _eventList.push_back(e);
}

void eCantorixSynth::lyricize(QString lyric)
{
    _eventList.clear();
    _wavtool.clear();
    bool inPho=false;
    QString pho;
    for(int i = 0; i< lyric.length(); i++)
    {
       QString i0 = lyric.at(i);

       if(i0=="["){
           inPho=true;
           continue;
       }
       if(i0=="]"){
           inPho=false;
           continue;
       }
       if(inPho)
       {
           pho+=i0;
       }
    }
    QStringList phoList = pho.split(" ");
    for(int i=0;i<phoList.count();i++)
    {
        pho = phoList[i];
        lookupPho(pho);
        //lookup pho, store result in list
    }
}

float eCantorixSynth::getPreutter()
{
        int preutter=0;
        for(int i=0;i<_eventList.count();i++)
        {
            phoEvent e=_eventList[i];
            preutter += e.type.defaultLength;
            //DEVLOG_DEBUG("l: "+e.symbol);
            if(e.type.defaultLength==0) break;
        }
        return 0.001*preutter;
}

QString genPitch(float pitch)
{
    int p=pitch;
    return "0 "+STR(p)+ " 100 "+STR(p);
}

bool eCantorixSynth::synthesize(IScore *score)
{
    _notes.clear();
    for(int i=0;i<score->getNoteCount();i++)
    {
        auto n = score->getNote(i);
        //DEVLOG_DEBUG("note "+STR(i));
        //DEVLOG_DEBUG("rest "+STR(n.rest)+" length "+STR(n.lenght));
        lyricize(n.lyric);
        eCantorixNote en;
        en.vstart=n.start;
        en.rlength=n.lenght;
        en.f0=frequencyFromNote(n.pitch);
        en.preutter = getPreutter();
        DEVLOG_DEBUG("preutter "+STR(en.preutter));
        en.eventList= _eventList;
        _notes.push_back(en);
    }
    for(int i=1;i<score->getNoteCount();i++)
    {

        float a = _notes[i-1].vstart + _notes[i-1].rlength;
        float b = _notes[i].vstart - _notes[i].preutter;
        if(b<a)
        {
           _notes[i-1].rlength -= _notes[i].preutter;
        }
    }

    for(int i=0;i<score->getNoteCount();i++)
    {
        _eventList = _notes[i].eventList;
        float f0 = _notes[i].f0;

       float fixedLength=0;
       int zeros=0;
       DEVLOG_DEBUG("rlength: "+STR(_notes[i].rlength));
       for(int j=0;j<_eventList.count();j++)
       {
           phoEvent e=_eventList[j];
           fixedLength += e.type.defaultLength;
           if(e.type.defaultLength==0) zeros++;
       }
       fixedLength /= 1000.0;
       float totalLength = _notes[i].preutter+_notes[i].rlength;
       if(totalLength>fixedLength && zeros>0)
       {
           QString pho;
           for(int j=0;j<_eventList.count();j++)
           {
               phoEvent e=_eventList[j];
               int len = e.type.defaultLength;
               bool sustained=false;
               if(len==0)
               {
                   len=(totalLength-fixedLength)*1000/zeros;
                   sustained=true;
               }
               int lmax=150;
               if(sustained && len>lmax)
               {
                   int len2 = len % lmax;
                   int len3 = len2/2;
                   int count = len/lmax;
                   len2-=len3;
                   DEVLOG_DEBUG("len2 "+STR(len2));
                   DEVLOG_DEBUG("len3 "+STR(len3));
                   DEVLOG_DEBUG("count "+STR(count));
                   assert(count*lmax+len2+len3==len);

                   pho += e.symbol;
                   pho += "\t";
                   pho += STR(len2);
                   pho += "\t";
                   pho += genPitch(f0);
                   pho += "\n";

                   for(int k=0;k<count;k++)
                   {
                       pho += e.symbol;
                       pho += "\t";
                       pho += STR(lmax);
                       pho += "\t";
                       pho += genPitch(f0);
                       pho += "\n";
                   }

                   pho += e.symbol;
                   pho += "\t";
                   pho += STR(len3);
                   pho += "\t";
                   pho += genPitch(f0);
                   pho += "\n";


               }
               else
               {
               pho += e.symbol;
               pho += "\t";
               pho += STR(len);
               pho += "\t";
               pho += genPitch(f0);
               pho += "\n";
               }
           }
           _notes[i].mbrolaPho = pho;
       }

    }
    DEVLOG_DEBUG("voice "+_internalVoice);
    QString cacheDir="/tmp/ecantorix_mbrola_"+_internalVoice;
    QDir dir(cacheDir);
    if (!dir.exists())
        dir.mkpath(".");
    chdir(cacheDir.toUtf8());
    for(int i=0;i<score->getNoteCount();i++)
    {
        QString basename = STR(i) +"_"+ QString(QCryptographicHash::hash(_notes[i].mbrolaPho.toUtf8(),QCryptographicHash::Md5).toHex());
        //DEVLOG_DEBUG("send_to_mbrola\n"+_notes[i].mbrolaPho);
        DEVLOG_DEBUG("basename \n"+basename);
        {
            QString phoFileName = basename+".pho";
            QString wavFileName = basename+".wav";
            QString tmpFileName = STR(i)+".wav";
            QFile file (phoFileName);
            if(file.open(QIODevice::WriteOnly))
            {
                QTextStream outstream(&file);
                outstream << _notes[i].mbrolaPho;
            }
            else
                abort();
            QString voice = "/home/isengaara/Hacking/Audio/ecantorix-sinsy-ng/mbrola-voices/data/"+_internalVoice+"/"+_internalVoice;
            QString mbrola = "mbrola -e "+voice+" "+phoFileName+" "+wavFileName+" 2> mbrola.log";
            DEVLOG_DEBUG(mbrola);
            system(mbrola.toUtf8());
            QFile logfile("mbrola.log");
            if(logfile.open(QIODevice::ReadOnly))
            {
                QTextStream instream(&logfile);
                QString text;
                do
                {
                    text = instream.readLine();
                    emit logError(text);

                } while(text.length());

            }
            else
                abort();


            _wavtool.addNote(wavFileName.toUtf8(),_notes[i].vstart-_notes[i].preutter,_notes[i].preutter+_notes[i].rlength);

            //agc(,tmpFileName.toUtf8());

        }


        //store pho files in cache dir
    }
    _wavtool.mix();
    _ctrl->startOfflinePlayback("/tmp/wavtool.wav");
    return true;
}


int eCantorixSynth::readData(float *data, int size)
{
    abort();
    (void)data;
    return size;
}


/// phoneme transformation

QString eCantorixSynth::getTranscription(QString txt)
{
    return txt;
}


bool eCantorixSynth::doPhonemeTransformation(QStringList& list)
{
    //TODO lyrizer
    DEVLOG_DEBUG(STR(list.count()));
    return false;
}

/// voice list (fix this there is only one)

bool eCantorixSynth::setVoice(QString voiceName)
{
    if(_voices.keys().contains(voiceName))
    {
        _internalVoice = _voices[voiceName];
        return true;
    }
    else
    {
        _internalVoice = "";
        return false;
    }
}


QStringList eCantorixSynth::listVoices()
{
    return _voices.keys();
}

/// logging (helper) (refactor this)
void eCantorixSynth::on_logDebug(QString debug)
{
    _ctrl->logDebug(debug);
}

void eCantorixSynth::on_logError(QString error)
{
    _ctrl->logError(error);
}

void eCantorixSynth::on_logSuccess(QString success)
{
    _ctrl->logSuccess(success);
}

//remove cacheDir support
//FIX no sound