sekai/src/MBRSynth.cpp

#include "sekai/MBRSynth.h"
#include "sekai/Track.h"

#include <string.h>
#include <fstream>
#include <iostream>
#include <sstream>

#include <json/json.h>
#include <math.h>
#include <sndfile.h>
#include <boost/filesystem.hpp>

#include <sekai/VoiceDefESPEAK.h>

class VoiceDefMBR : public VoiceDef {
private:
    float *_input_data=nullptr;
    int _input_data_length;
    int _input_data_length_ref;
    int _samplerate;
public:
    VoiceDefMBR(std::string path,std::string ref)
    {
        SF_INFO info = {0};
        SNDFILE *sf = sf_open(path.c_str(), SFM_READ, &info);
        if (sf == nullptr) {
            return;  // false
        }
        _input_data = new float[info.frames];
        _input_data_length = info.frames;
        assert(info.channels == 1);
        _samplerate = info.samplerate;
        sf_read_float(sf, _input_data, info.frames);
        sf_close(sf);
        
        if(ref.length())
        {
            SNDFILE *sf = sf_open(ref.c_str(), SFM_READ, &info);
            assert(_samplerate==info.samplerate);
            _input_data_length_ref = info.frames;
            if(sf) sf_close(sf);
        }
    }

    virtual float getLength() {
        if(_input_data_length_ref)
            return _input_data_length_ref*1.0/_samplerate;
        else
            return _input_data_length*1.0/_samplerate;
    }
    virtual void getImpulseResponse(float pos, float *impulseResponse, int *impulseResponseLength, float morph) {


        //FIXME: do not hardcode
        float frame_period = 5.0;
        int fft_size = 2048;

        float fract = pos * 1000 / frame_period;
        int frame_index = (int)fract;
        int frame_offset_l = frame_index * fft_size;
        int frame_offset_r = (frame_index+1) * fft_size;
        float pmk_interp = fract-frame_index;

        if(frame_offset_l<0 || frame_offset_r<0) return;

        for(int i=0; i<fft_size; i++)
        {
            float x=0;
            float l=0;
            float r=0;
            if(i+frame_offset_l<_input_data_length)
                l = _input_data[i+frame_offset_l];
            if(i+frame_offset_r<_input_data_length)
                r = _input_data[i+frame_offset_r];

            x = r * (1 - pmk_interp) + l * pmk_interp;

            if (morph) {
                r = x;
                l = impulseResponse[i];
                x = r * morph + l * (1.0f - morph);
            }
            
            impulseResponse[i] = x;
        }
        
        *impulseResponseLength=fft_size;
    }
    virtual int getSamplerate() {
        return _samplerate;
    }
    virtual std::string getPhoLine(int index) {
        return "";
    }

};



MBRSynth::MBRSynth(ControlTrack *ctrl, int buffer_size)
    : VoiceSampler(buffer_size) {
    _impulseResponse = new float[IMPULSE_RESPONSE_MAX];
    _ctrl = ctrl;
    _samplerate = 0;
    _debugfile = fopen("/tmp/maxvol.txt","w");
}

void MBRSynth::addUnit(const std::string &lyric, int count, float *a,float *b) {

    load(lyric);


    PhoEvent *e = new PhoEvent;
    e->points = count;
    e->voice = _voicemap[lyric];

    for (int i = 0; i < count; i++) {
        e->x[i] = a[i];
        e->y[i] = b[i];
    }
    _phoEvents.addEvent(e);
}

bool MBRSynth::addOnePulse() {
    float currentTime = inputPositionSamples() / _samplerate;

    if(isnan(currentTime))
    {
        printf("MBRSynth: samplerate is invalid\n");
        return false;
    }

    _phoEvents.selectNext(currentTime);


    PhoEvent *pho0 = _phoEvents.current();
    PhoEvent *pho1 = _phoEvents.next();

    if (pho0 == nullptr) return false;

    assert(_ctrl);

    float output_f0= _ctrl->getF0atTime(currentTime);


    if (output_f0 == 0) output_f0 = 50;

    if (currentTime < pho0->start()) {
        // rest: output silence
        output_f0 = 500;
        float period = _samplerate * 1.0f / output_f0;
        float dummy;
        ola(&dummy, 0, period);
        return true;
    }

    if (currentTime >= pho0->start() && currentTime < pho0->end()) {
        float interp = 0;
        int impulseResponseLength = 0;

        getImpulseResponse(currentTime, pho0, _impulseResponse,
                           &impulseResponseLength,
                           0);  // TODO get impulse response from mapped index

        if (pho1 && currentTime >= pho1->start()) {
            interp = (currentTime - pho1->start()) / (pho0->end() - pho1->start());
        }

        if (interp > 0) {
            getImpulseResponse(currentTime, pho1, _impulseResponse,
                               &impulseResponseLength,
                               interp);  // needs interp as input
        }

        // int tmp = static_cast<int>(_samplerate * 1.0f / output_f0);
        // float period = tmp;
        float period = _samplerate * 1.0f / output_f0;

        VoiceSampler::hanningWindow(_impulseResponse, impulseResponseLength);

        ola(_impulseResponse, impulseResponseLength, period);
        return true;
    }
    {
        // rest: output silence
        output_f0 = 500;
        float period = _samplerate * 1.0f / output_f0;
        float dummy;
        ola(&dummy, 0, period);
        return true;
    }
    return false;
}

void MBRSynth::postProcess(float* data, int size)
{
	float thresh=0.95;
    for(int i=0;i<size;i++)
    {
        float t = (i + outputPos)*1.0/_samplerate;
        data[i] *= _ctrl->getDynamicsAtTime(t); 
        float vol = fabs(data[i]);
        if(vol > _maxvol) _maxvol = vol;
        if(_maxvol > thresh)
        {
			float a = thresh/_maxvol;
			data[i] *= a;
		}
    }
    fprintf(_debugfile,"maxvol = %f\n",_maxvol);
    
}

void MBRSynth::getImpulseResponse(float currentTime, PhoEvent *event,
                                  float *impulseResponse,
                                  int *impulseResponseLength, float morph) {
    auto voice = event->voice;
    float localTime =
        interp_linear(event->x, event->y, event->points, currentTime);

    assert(voice);
    if(voice==nullptr) return;
    voice->getImpulseResponse(localTime,impulseResponse,impulseResponseLength,morph);
    //TODO: morph is ignored
}

float MBRSynth::getLengthForUnit(const std::string &fileName) {

    load(fileName);
    return _voicemap[fileName]->getLength();
}

std::string MBRSynth::getPhoLine(const std::string &fileName, int index)
{
    load(fileName);
    return _voicemap[fileName]->getPhoLine(index);
}

void MBRSynth::load(const std::string &unit)
{

    if (_voicemap[unit] == nullptr) {
        
        synthType type = synthType::FVOX;
        if(_config) type = _config->type;

        switch(type)
        {
            case synthType::UTAU:
            { 
                std::string debug = _basedir+"/"+unit+".ogg";
                _voicemap[unit] = new VoiceDefMBR(_basedir+"/"+unit+".ogg",_basedir+"/"+unit+".wav");
                break;
            }
            case synthType::ESPEAK:
            {
                _voicemap[unit] = new VoiceDefESPEAK(unit);
                break;
            }
            case synthType::FVOX:
            {
                printf("VoiceDefFOX %s %s\n",_basedir.c_str(),unit.c_str());
                _voicemap[unit] = new VoiceDefMBR(_basedir+"/ogg/"+unit+".ogg",_basedir+"/wav/"+unit+".wav");
                break;
            }
            default:
            {
                printf("MBRSynth::load failed %s\n",unit.c_str());
                throw new std::runtime_error("unknown voice type");
            }
        }
        
        if(_samplerate==0) {
            _samplerate=_voicemap[unit]->getSamplerate();
        }
        else
            assert(_samplerate==_voicemap[unit]->getSamplerate());
    }
}

bool MBRSynth::saveWave(const std::string &filename)
{
    SF_INFO info = {0};
    info.samplerate = _samplerate;
    info.channels = 1;
    info.format = SF_FORMAT_WAV | SF_FORMAT_PCM_16;
    SNDFILE *sf = sf_open(filename.c_str(), SFM_WRITE, &info);
    if(sf==nullptr) return false;

    while (1) {
        const int size = 1024;
        int fill = size * 4;

        float buffer_out[size];
        if (readData(buffer_out, size, fill) == false) break;
        sf_write_float(sf, buffer_out, size);
    }

    sf_close(sf);
    return true;
}