Implemented MP3 encoder and necessary infrastructure around it

This commit is contained in:
vpereverzev 2022-04-20 17:06:24 +02:00 committed by pereverzev+v
parent bc987bf662
commit 87902afac5
24 changed files with 581 additions and 12 deletions

View file

@ -159,6 +159,15 @@ set(MODULE_SRC
${CMAKE_CURRENT_LIST_DIR}/view/synthssettingsmodel.cpp
${CMAKE_CURRENT_LIST_DIR}/view/synthssettingsmodel.h
# Encoders
${CMAKE_CURRENT_LIST_DIR}/internal/encoders/abstractaudioencoder.h
${CMAKE_CURRENT_LIST_DIR}/internal/encoders/mp3encoder.cpp
${CMAKE_CURRENT_LIST_DIR}/internal/encoders/mp3encoder.h
# SoundTracks
${CMAKE_CURRENT_LIST_DIR}/internal/soundtracks/soundtrackwriter.cpp
${CMAKE_CURRENT_LIST_DIR}/internal/soundtracks/soundtrackwriter.h
# DevTools
${CMAKE_CURRENT_LIST_DIR}/devtools/waveformmodel.cpp
${CMAKE_CURRENT_LIST_DIR}/devtools/waveformmodel.h

View file

@ -74,14 +74,12 @@ protected:
{
for (const auto& pair : events) {
for (const mpe::PlaybackEvent& event : pair.second) {
mpe::timestamp_t actualTimestamp = 0;
if (std::holds_alternative<mpe::NoteEvent>(event)) {
actualTimestamp = std::get<mpe::NoteEvent>(event).arrangementCtx().actualTimestamp;
} else {
actualTimestamp = std::get<mpe::RestEvent>(event).arrangementCtx().actualTimestamp;
if (!std::holds_alternative<mpe::NoteEvent>(event)) {
continue;
}
mpe::timestamp_t actualTimestamp = std::get<mpe::NoteEvent>(event).arrangementCtx().actualTimestamp;
m_events[actualTimestamp].emplace_back(event);
}
}

View file

@ -37,7 +37,9 @@
namespace mu::audio {
using msecs_t = int64_t;
using secs_t = int64_t;
using samples_t = uint64_t;
using sample_rate_t = uint64_t;
using audioch_t = uint8_t;
using volume_db_t = float;
using volume_dbfs_t = float;
@ -270,6 +272,28 @@ enum class PlaybackStatus {
Paused,
Running
};
enum class SoundTrackType {
Undefined = -1,
MP3,
OGG,
FLAC
};
struct SoundTrackFormat {
SoundTrackType type = SoundTrackType::Undefined;
sample_rate_t sampleRate = 0;
audioch_t audioChannelsNumber = 0;
int bitRate = 0;
bool operator==(const SoundTrackFormat& other) const
{
return type == other.type
&& sampleRate == other.sampleRate
&& audioChannelsNumber == other.audioChannelsNumber
&& bitRate == other.bitRate;
}
};
}
#endif // MU_AUDIO_AUDIOTYPES_H

View file

@ -48,6 +48,8 @@ public:
virtual async::Promise<AudioSignalChanges> signalChanges(const TrackSequenceId sequenceId, const TrackId trackId) const = 0;
virtual async::Promise<AudioSignalChanges> masterSignalChanges() const = 0;
virtual async::Promise<bool> saveSoundTrack(const TrackSequenceId sequenceId, const io::path& destination, const SoundTrackFormat& format) = 0;
};
using IAudioOutputPtr = std::shared_ptr<IAudioOutput>;

View file

@ -0,0 +1,60 @@
/*
* SPDX-License-Identifier: GPL-3.0-only
* MuseScore-CLA-applies
*
* MuseScore
* Music Composition & Notation
*
* Copyright (C) 2021 MuseScore BVBA and others
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 3 as
* published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
#ifndef MU_AUDIO_ABSTRACTAUDIOENCODER_H
#define MU_AUDIO_ABSTRACTAUDIOENCODER_H
#include "log.h"
#include "audiotypes.h"
namespace mu::audio::encode {
template<class T>
class AbstractAudioEncoder
{
public:
static size_t requiredOutputBufferSize(samples_t samplesPerChannel)
{
return T::outputBufferSize(samplesPerChannel);
}
static samples_t encode(const SoundTrackFormat& format, samples_t samplesPerChannel, float* input, char* output)
{
IF_ASSERT_FAILED(input && output) {
return 0;
}
return T::doEncode(format, samplesPerChannel, input, output);
}
static samples_t flush(char* output, size_t outputSize)
{
IF_ASSERT_FAILED(output && outputSize > 0) {
return 0;
}
return T::doFlush(output, outputSize);
}
};
}
#endif // MU_AUDIO_ABSTRACTAUDIOENCODER_H

View file

@ -0,0 +1,114 @@
/*
* SPDX-License-Identifier: GPL-3.0-only
* MuseScore-CLA-applies
*
* MuseScore
* Music Composition & Notation
*
* Copyright (C) 2021 MuseScore BVBA and others
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 3 as
* published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
#include "mp3encoder.h"
#include "lame.h"
using namespace mu;
using namespace mu::audio;
using namespace mu::audio::encode;
struct LameHandler
{
static LameHandler* instance()
{
static LameHandler handler;
return &handler;
}
bool updateSpec(const SoundTrackFormat& format)
{
if (!isValid()) {
return false;
}
if (m_format == format) {
return true;
}
lame_set_num_channels(flags, m_format.audioChannelsNumber);
lame_set_in_samplerate(flags, m_format.sampleRate);
lame_set_brate(flags, m_format.bitRate);
return lame_init_params(flags) != 0;
}
const SoundTrackFormat& format() const
{
return m_format;
}
bool isValid() const
{
return flags;
}
lame_global_flags* flags = nullptr;
private:
LameHandler()
{
flags = lame_init();
lame_set_errorf(flags, [](const char* msg, va_list /*ap*/) {
LOGE() << msg;
});
lame_set_debugf(flags, [](const char* msg, va_list /*ap*/) {
LOGD() << msg;
});
lame_set_msgf(flags, [](const char* msg, va_list /*ap*/) {
LOGI() << msg;
});
}
~LameHandler()
{
lame_close(flags);
}
SoundTrackFormat m_format;
};
size_t Mp3Encoder::outputBufferSize(samples_t samplesPerChannel)
{
//!Note See thirdparty/lame/API
return 1.25 * samplesPerChannel + 7200;
}
samples_t Mp3Encoder::doEncode(const SoundTrackFormat& format,
samples_t samplesPerChannel, float* input, char* output)
{
LameHandler::instance()->updateSpec(format);
return lame_encode_buffer_interleaved_ieee_float(LameHandler::instance()->flags, input, samplesPerChannel,
reinterpret_cast<unsigned char*>(output),
samplesPerChannel * format.audioChannelsNumber);
}
samples_t Mp3Encoder::doFlush(char* output, size_t outputSize)
{
return lame_encode_flush(LameHandler::instance()->flags,
reinterpret_cast<unsigned char*>(output),
outputSize);
}

View file

@ -0,0 +1,40 @@
/*
* SPDX-License-Identifier: GPL-3.0-only
* MuseScore-CLA-applies
*
* MuseScore
* Music Composition & Notation
*
* Copyright (C) 2021 MuseScore BVBA and others
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 3 as
* published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
#ifndef MU_AUDIO_MP3ENCODER_H
#define MU_AUDIO_MP3ENCODER_H
#include "abstractaudioencoder.h"
namespace mu::audio::encode {
class Mp3Encoder : public AbstractAudioEncoder<Mp3Encoder>
{
protected:
friend class AbstractAudioEncoder;
static size_t outputBufferSize(samples_t samplesPerChannel);
static samples_t doEncode(const SoundTrackFormat& format, samples_t samplesPerChannel, float* input, char* output);
static samples_t doFlush(char* output, size_t outputSize);
};
}
#endif // MU_AUDIO_MP3ENCODER_H

View file

@ -0,0 +1,154 @@
/*
* SPDX-License-Identifier: GPL-3.0-only
* MuseScore-CLA-applies
*
* MuseScore
* Music Composition & Notation
*
* Copyright (C) 2021 MuseScore BVBA and others
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 3 as
* published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
#include "soundtrackwriter.h"
#include "internal/worker/audioengine.h"
#include "internal/encoders/mp3encoder.h"
using namespace mu;
using namespace mu::audio;
using namespace mu::audio::soundtrack;
static constexpr audioch_t SUPPORTED_AUDIO_CHANNELS_COUNT = 2;
static constexpr samples_t SAMPLES_PER_CHANNEL = 2048;
static constexpr size_t INTERNAL_BUFFER_SIZE = SUPPORTED_AUDIO_CHANNELS_COUNT * SAMPLES_PER_CHANNEL;
SoundTrackWriter::SoundTrackWriter(const io::path& destination, const SoundTrackFormat& format, const msecs_t totalDuration, IAudioSourcePtr source)
: m_format(format),
m_source(std::move(source))
{
m_fileStream = std::fopen(destination.c_str(), "a+");
if (!m_fileStream || !m_source) {
return;
}
samples_t totalSamplesNumber = (totalDuration / 1000.f) * source->audioChannelsCount() * format.sampleRate;
m_inputBuffer.resize(totalSamplesNumber);
m_intermBuffer.resize(INTERNAL_BUFFER_SIZE);
m_outputBuffer.resize(requiredOutputBufferSize(format.type, totalSamplesNumber));
}
SoundTrackWriter::~SoundTrackWriter()
{
std::fclose(m_fileStream);
}
bool SoundTrackWriter::write()
{
TRACEFUNC;
if (m_format.type == SoundTrackType::Undefined || !m_source) {
return false;
}
if (!m_fileStream) {
return false;
}
m_source->setSampleRate(m_format.sampleRate);
m_source->setIsActive(true);
if (!prepareInputBuffer()) {
return false;
}
if (!writeEncodedOutput()) {
return false;
}
completeOutput();
m_source->setSampleRate(AudioEngine::instance()->sampleRate());
m_source->setIsActive(false);
return true;
}
SoundTrackWriter::EncodeFunc SoundTrackWriter::encodeHandler() const
{
switch (m_format.type) {
case SoundTrackType::MP3: return encode::Mp3Encoder::encode;
case SoundTrackType::OGG: return nullptr;
case SoundTrackType::FLAC: return nullptr;
default: return nullptr;
}
}
size_t SoundTrackWriter::requiredOutputBufferSize(const SoundTrackType type, const samples_t samplesPerChannel) const
{
switch (type) {
case SoundTrackType::MP3: return encode::Mp3Encoder::requiredOutputBufferSize(samplesPerChannel);
case SoundTrackType::OGG: return 0;
case SoundTrackType::FLAC: return 0;
default: return 0;
}
}
bool SoundTrackWriter::prepareInputBuffer()
{
size_t inputBufferOffset = 0;
size_t inputBufferMaxOffset = m_inputBuffer.size();
while (m_source->process(m_intermBuffer.data(), SAMPLES_PER_CHANNEL) != 0) {
if (inputBufferOffset >= inputBufferMaxOffset) {
break;
}
size_t samplesToCopy = std::min(INTERNAL_BUFFER_SIZE, inputBufferMaxOffset - inputBufferOffset);
std::copy(m_intermBuffer.begin(),
m_intermBuffer.begin() + samplesToCopy,
m_inputBuffer.begin() + inputBufferOffset);
inputBufferOffset += samplesToCopy;
}
if (inputBufferOffset == 0) {
LOGI() << "No audio to export";
return false;
}
return true;
}
bool SoundTrackWriter::writeEncodedOutput()
{
EncodeFunc encodeFunc = encodeHandler();
if (!encodeFunc) {
return false;
}
samples_t encodedBytes = encodeFunc(m_format, m_inputBuffer.size() / SUPPORTED_AUDIO_CHANNELS_COUNT,
m_inputBuffer.data(), m_outputBuffer.data());
std::fwrite(m_outputBuffer.data(), sizeof(char), encodedBytes, m_fileStream);
return true;
}
void SoundTrackWriter::completeOutput()
{
samples_t encodedBytes = encode::Mp3Encoder::flush(m_outputBuffer.data(), m_outputBuffer.size());
std::fwrite(m_outputBuffer.data(), sizeof(char), encodedBytes, m_fileStream);
}

View file

@ -0,0 +1,62 @@
/*
* SPDX-License-Identifier: GPL-3.0-only
* MuseScore-CLA-applies
*
* MuseScore
* Music Composition & Notation
*
* Copyright (C) 2021 MuseScore BVBA and others
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 3 as
* published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
#ifndef MU_AUDIO_SOUNDTRACKWRITER_H
#define MU_AUDIO_SOUNDTRACKWRITER_H
#include <vector>
#include <functional>
#include <cstdio>
#include "audiotypes.h"
#include "iaudiosource.h"
#include "internal/encoders/abstractaudioencoder.h"
namespace mu::audio::soundtrack {
class SoundTrackWriter
{
public:
SoundTrackWriter(const io::path &destination, const SoundTrackFormat& format, const msecs_t totalDuration, IAudioSourcePtr source);
~SoundTrackWriter();
bool write();
private:
using EncodeFunc = std::function<samples_t(const SoundTrackFormat&, samples_t, float*, char*)>;
EncodeFunc encodeHandler() const;
size_t requiredOutputBufferSize(const SoundTrackType type, const samples_t samplesPerChannel) const;
bool prepareInputBuffer();
bool writeEncodedOutput();
void completeOutput();
SoundTrackFormat m_format;
IAudioSourcePtr m_source = nullptr;
std::FILE* m_fileStream = nullptr;
std::vector<float> m_inputBuffer;
std::vector<float> m_intermBuffer;
std::vector<char> m_outputBuffer;
};
}
#endif // MU_AUDIO_SOUNDTRACKWRITER_H

View file

@ -81,6 +81,13 @@ void AudioEngine::deinit()
}
}
sample_rate_t AudioEngine::sampleRate() const
{
ONLY_AUDIO_WORKER_THREAD;
return m_sampleRate;
}
void AudioEngine::setSampleRate(unsigned int sampleRate)
{
ONLY_AUDIO_WORKER_THREAD;
@ -89,6 +96,11 @@ void AudioEngine::setSampleRate(unsigned int sampleRate)
return;
}
if (m_sampleRate == sampleRate) {
return;
}
m_sampleRate = sampleRate;
m_mixer->mixedSource()->setSampleRate(sampleRate);
}

View file

@ -43,6 +43,8 @@ public:
Ret init(IAudioBufferPtr bufferPtr);
void deinit();
sample_rate_t sampleRate() const;
void setSampleRate(unsigned int sampleRate);
void setReadBufferSize(uint16_t readBufferSize);
void setAudioChannelsCount(const audioch_t count);
@ -54,6 +56,8 @@ private:
bool m_inited = false;
sample_rate_t m_sampleRate = 0;
MixerPtr m_mixer = nullptr;
IAudioBufferPtr m_buffer = nullptr;
};

View file

@ -27,11 +27,13 @@
#include "internal/audiosanitizer.h"
#include "internal/audiothread.h"
#include "internal/soundtracks/soundtrackwriter.h"
#include "internal/worker/audioengine.h"
#include "audioerrors.h"
using namespace mu::audio;
using namespace mu::async;
using namespace mu::audio::soundtrack;
AudioOutputHandler::AudioOutputHandler(IGetTrackSequence* getSequence)
: m_getSequence(getSequence)
@ -158,6 +160,28 @@ Promise<AudioSignalChanges> AudioOutputHandler::masterSignalChanges() const
}, AudioThread::ID);
}
Promise<bool> AudioOutputHandler::saveSoundTrack(const TrackSequenceId sequenceId, const io::path& destination,
const SoundTrackFormat& format)
{
return Promise<bool>([this, sequenceId, destination, format](auto resolve, auto reject) {
ONLY_AUDIO_WORKER_THREAD;
IF_ASSERT_FAILED(mixer()) {
return reject(static_cast<int>(Err::Undefined), "undefined reference to a mixer");
}
ITrackSequencePtr s = sequence(sequenceId);
if (!s) {
return reject(static_cast<int>(Err::InvalidSequenceId), "invalid sequence id");
}
msecs_t totalDuration = s->player()->duration();
SoundTrackWriter writer(destination, format, totalDuration, mixer());
return resolve(writer.write());
}, AudioThread::ID);
}
std::shared_ptr<Mixer> AudioOutputHandler::mixer() const
{
return AudioEngine::instance()->mixer();

View file

@ -52,6 +52,9 @@ public:
async::Promise<AudioSignalChanges> signalChanges(const TrackSequenceId sequenceId, const TrackId trackId) const override;
async::Promise<AudioSignalChanges> masterSignalChanges() const override;
async::Promise<bool> saveSoundTrack(const TrackSequenceId sequenceId, const io::path& destination,
const SoundTrackFormat& format) override;
private:
std::shared_ptr<Mixer> mixer() const;
ITrackSequencePtr sequence(const TrackSequenceId id) const;

View file

@ -106,6 +106,11 @@ void Clock::seek(const msecs_t msecs)
m_seekOccurred.notify();
}
msecs_t Clock::timeDuration() const
{
return m_timeDuration;
}
void Clock::setTimeDuration(const msecs_t duration)
{
m_timeDuration = duration;

View file

@ -43,6 +43,7 @@ public:
void resume() override;
void seek(const msecs_t msecs) override;
msecs_t timeDuration() const override;
void setTimeDuration(const msecs_t duration) override;
Ret setTimeLoop(const msecs_t fromMsec, const msecs_t toMsec) override;
void resetTimeLoop() override;

View file

@ -48,6 +48,7 @@ public:
virtual void resume() = 0;
virtual void seek(const msecs_t msecs) = 0;
virtual msecs_t timeDuration() const = 0;
virtual void setTimeDuration(const msecs_t duration) = 0;
virtual Ret setTimeLoop(const msecs_t fromMsec, const msecs_t toMsec) = 0;
virtual void resetTimeLoop() = 0;

View file

@ -40,6 +40,7 @@ public:
virtual void pause() = 0;
virtual void resume() = 0;
virtual msecs_t duration() const = 0;
virtual void setDuration(const msecs_t duration) = 0;
virtual Ret setLoop(const msecs_t fromMsec, const msecs_t toMsec) = 0;
virtual void resetLoop() = 0;

View file

@ -154,6 +154,17 @@ samples_t Mixer::process(float* outBuffer, samples_t samplesPerChannel)
return masterChannelSampleCount;
}
void Mixer::setIsActive(bool arg)
{
ONLY_AUDIO_WORKER_THREAD;
AbstractAudioSource::setIsActive(arg);
for (const auto& channel : m_mixerChannels) {
channel.second->setIsActive(arg);
}
}
void Mixer::addClock(IClockPtr clock)
{
ONLY_AUDIO_WORKER_THREAD;

View file

@ -62,6 +62,7 @@ public:
void setSampleRate(unsigned int sampleRate) override;
unsigned int audioChannelsCount() const override;
samples_t process(float* outBuffer, samples_t samplesPerChannel) override;
void setIsActive(bool arg) override;
private:
void mixOutputFromChannel(float* outBuffer, float* inBuffer, unsigned int samplesCount);

View file

@ -106,14 +106,22 @@ bool MixerChannel::isActive() const
{
ONLY_AUDIO_WORKER_THREAD;
return !m_params.muted;
IF_ASSERT_FAILED(m_audioSource) {
return false;
}
return m_audioSource->isActive();
}
void MixerChannel::setIsActive(bool arg)
{
ONLY_AUDIO_WORKER_THREAD;
m_params.muted = !arg;
IF_ASSERT_FAILED(m_audioSource) {
return;
}
m_audioSource->setIsActive(arg);
}
void MixerChannel::setSampleRate(unsigned int sampleRate)

View file

@ -101,6 +101,17 @@ void SequencePlayer::resume()
}
}
msecs_t SequencePlayer::duration() const
{
ONLY_AUDIO_WORKER_THREAD;
if (!m_clock) {
return 0;
}
return m_clock->timeDuration();
}
void SequencePlayer::setDuration(const msecs_t duration)
{
ONLY_AUDIO_WORKER_THREAD;

View file

@ -41,6 +41,7 @@ public:
void pause() override;
void resume() override;
msecs_t duration() const override;
void setDuration(const msecs_t duration) override;
Ret setLoop(const msecs_t fromMsec, const msecs_t toMsec) override;
void resetLoop() override;

View file

@ -30,10 +30,27 @@ using namespace mu::framework;
mu::Ret Mp3Writer::write(notation::INotationPtr notation, io::Device& destinationDevice, const Options& options)
{
UNUSED(notation)
UNUSED(destinationDevice)
UNUSED(options)
NOT_IMPLEMENTED;
//!Note Temporary workaround, since io::Device is the alias for QIODevice, which falls with SIGSEGV
//! on any call from background thread. Once we have our own implementation of io::Device
//! we can pass io::Device directly into IPlayback::IAudioOutput::saveSoundTrack
QFile* file = qobject_cast<QFile*>(&destinationDevice);
return make_ret(Ret::Code::NotImplemented);
QFileInfo info(*file);
QString path = info.absoluteFilePath();
//TODO Take actual data
audio::TrackSequenceId currentSequenceId = 0;
audio::SoundTrackFormat format { audio::SoundTrackType::MP3, 44100, 2, 128 };
playback()->audioOutput()->saveSoundTrack(currentSequenceId, io::path(info.absoluteFilePath()), std::move(format))
.onResolve(this, [path](const bool /*result*/) {
LOGD() << "Successfully saved sound track by path: " << path;
})
.onReject(this, [](int errorCode, const std::string& msg) {
LOGE() << "errorCode: " << errorCode << ", " << msg;
});
return make_ret(Ret::Code::Ok);
}

View file

@ -23,11 +23,17 @@
#ifndef MU_IMPORTEXPORT_MP3WRITER_H
#define MU_IMPORTEXPORT_MP3WRITER_H
#include "modularity/ioc.h"
#include "audio/iplayback.h"
#include "audio/iaudiooutput.h"
#include "async/asyncable.h"
#include "abstractaudiowriter.h"
namespace mu::iex::audioexport {
class Mp3Writer : public AbstractAudioWriter
class Mp3Writer : public AbstractAudioWriter, public async::Asyncable
{
INJECT(audioexport, audio::IPlayback, playback)
public:
Ret write(notation::INotationPtr notation, io::Device& destinationDevice, const Options& options = Options()) override;
};