Implement mouth animation for NPCs based on say sound (Fixes #642)

apps/openmw/CMakeLists.txt

@@ -48,7 +48,7 @@ add_openmw_dir (mwscript
     )
 
 add_openmw_dir (mwsound
-    soundmanagerimp openal_output ffmpeg_decoder sound
+    soundmanagerimp openal_output ffmpeg_decoder sound sound_decoder sound_output loudness
     )
 
 add_openmw_dir (mwworld

apps/openmw/mwbase/soundmanager.hpp

@@ -101,6 +101,11 @@ namespace MWBase
             virtual void stopSay(const MWWorld::Ptr &reference=MWWorld::Ptr()) = 0;
             ///< Stop an actor speaking
 
+            virtual float getSaySoundLoudness(const MWWorld::Ptr& reference) const = 0;
+            ///< Check the currently playing say sound for this actor
+            /// and get an average loudness value (scale [0,1]) at the current time position.
+            /// If the actor is not saying anything, returns 0.
+
             virtual SoundPtr playTrack(const MWSound::DecoderPtr& decoder, PlayType type) = 0;
             ///< Play a 2D audio track, using a custom decoder
 

apps/openmw/mwrender/npcanimation.cpp

@@ -66,15 +66,40 @@ std::string getVampireHead(const std::string& race, bool female)
 namespace MWRender
 {
 
+HeadAnimationTime::HeadAnimationTime(MWWorld::Ptr reference)
+    : mReference(reference), mTalkStart(0), mTalkStop(0), mBlinkStart(0), mBlinkStop(0)
+{
+}
+
 float HeadAnimationTime::getValue() const
 {
-    // TODO use time from text keys (Talk Start/Stop, Blink Start/Stop)
     // TODO: Handle eye blinking
     if (MWBase::Environment::get().getSoundManager()->sayDone(mReference))
-        return 0;
+        return mBlinkStop;
     else
-        // TODO: Use the loudness of the currently playing sound
+        return mTalkStart +
-        return 1;
+            (mTalkStop - mTalkStart) *
+            std::min(1.f, MWBase::Environment::get().getSoundManager()->getSaySoundLoudness(mReference)*4); // Rescale a bit (most voices are not very loud)
+}
+
+void HeadAnimationTime::setTalkStart(float value)
+{
+    mTalkStart = value;
+}
+
+void HeadAnimationTime::setTalkStop(float value)
+{
+    mTalkStop = value;
+}
+
+void HeadAnimationTime::setBlinkStart(float value)
+{
+    mBlinkStart = value;
+}
+
+void HeadAnimationTime::setBlinkStop(float value)
+{
+    mBlinkStop = value;
 }
 
 static NpcAnimation::PartBoneMap createPartListMap()
@@ -620,7 +645,21 @@ bool NpcAnimation::addOrReplaceIndividualPart(ESM::PartReferenceType type, int g
             ctrl->setSource(mNullAnimationTimePtr);
 
             if (type == ESM::PRT_Head)
+            {
                 ctrl->setSource(mHeadAnimationTime);
+                const NifOgre::TextKeyMap& keys = mObjectParts[type]->mTextKeys;
+                for (NifOgre::TextKeyMap::const_iterator it = keys.begin(); it != keys.end(); ++it)
+                {
+                    if (Misc::StringUtils::ciEqual(it->second, "talk: start"))
+                        mHeadAnimationTime->setTalkStart(it->first);
+                    if (Misc::StringUtils::ciEqual(it->second, "talk: stop"))
+                        mHeadAnimationTime->setTalkStop(it->first);
+                    if (Misc::StringUtils::ciEqual(it->second, "blink: start"))
+                        mHeadAnimationTime->setBlinkStart(it->first);
+                    if (Misc::StringUtils::ciEqual(it->second, "blink: stop"))
+                        mHeadAnimationTime->setBlinkStop(it->first);
+                }
+            }
             else if (type == ESM::PRT_Weapon)
                 ctrl->setSource(mWeaponAnimationTime);
         }

apps/openmw/mwrender/npcanimation.hpp

@@ -19,8 +19,17 @@ class HeadAnimationTime : public Ogre::ControllerValue<Ogre::Real>
 {
 private:
     MWWorld::Ptr mReference;
+    float mTalkStart;
+    float mTalkStop;
+    float mBlinkStart;
+    float mBlinkStop;
 public:
-    HeadAnimationTime(MWWorld::Ptr reference) : mReference(reference) {}
+    HeadAnimationTime(MWWorld::Ptr reference);
+
+    void setTalkStart(float value);
+    void setTalkStop(float value);
+    void setBlinkStart(float value);
+    void setBlinkStop(float value);
 
     virtual Ogre::Real getValue() const;
     virtual void setValue(Ogre::Real value)

apps/openmw/mwsound/loudness.cpp

@@ -0,0 +1,53 @@
+#include "loudness.hpp"
+
+#include "soundmanagerimp.hpp"
+
+namespace MWSound
+{
+
+    void analyzeLoudness(const std::vector<char> &data, int sampleRate, ChannelConfig chans,
+                         SampleType type, std::vector<float> &out, float valuesPerSecond)
+    {
+        int samplesPerSegment = sampleRate / valuesPerSecond;
+        int numSamples = bytesToFrames(data.size(), chans, type);
+        int advance = framesToBytes(1, chans, type);
+
+        out.reserve(numSamples/samplesPerSegment);
+
+        int segment=0;
+        int sample=0;
+        while (segment < numSamples/samplesPerSegment)
+        {
+            float sum=0;
+            int samplesAdded = 0;
+            while (sample < numSamples && sample < (segment+1)*samplesPerSegment)
+            {
+                // get sample on a scale from -1 to 1
+                float value = 0;
+                if (type == SampleType_UInt8)
+                    value = data[sample*advance]/128.f;
+                else if (type == SampleType_Int16)
+                {
+                    value = *reinterpret_cast<const Ogre::int16*>(&data[sample*advance]);
+                    value /= float(std::numeric_limits<Ogre::uint16>().max());
+                }
+                else if (type == SampleType_Float32)
+                {
+                    value = *reinterpret_cast<const float*>(&data[sample*advance]);
+                    value /= std::numeric_limits<float>().max();
+                }
+
+                sum += value*value;
+                ++samplesAdded;
+                ++sample;
+            }
+
+            float rms = 0; // root mean square
+            if (samplesAdded > 0)
+                rms = std::sqrt(sum / samplesAdded);
+            out.push_back(rms);
+            ++segment;
+        }
+    }
+
+}

apps/openmw/mwsound/loudness.hpp

@@ -0,0 +1,20 @@
+#include "sound_decoder.hpp"
+
+namespace MWSound
+{
+
+/**
+ * Analyzes the energy (closely related to loudness) of a sound buffer.
+ * The buffer will be divided into segments according to \a valuesPerSecond,
+ * and for each segment a loudness value in the range of [0,1] will be computed.
+ * @param data the sound buffer to analyze, containing raw samples
+ * @param sampleRate the sample rate of the sound buffer
+ * @param chans channel layout of the buffer
+ * @param type sample type of the buffer
+ * @param out Will contain the output loudness values.
+ * @param valuesPerSecond How many loudness values per second of audio to compute.
+ */
+void analyzeLoudness (const std::vector<char>& data, int sampleRate, ChannelConfig chans, SampleType type,
+                      std::vector<float>& out, float valuesPerSecond);
+
+}

apps/openmw/mwsound/openal_output.cpp

@@ -11,11 +11,16 @@
 #include "sound_decoder.hpp"
 #include "sound.hpp"
 #include "soundmanagerimp.hpp"
+#include "loudness.hpp"
 
 #ifndef ALC_ALL_DEVICES_SPECIFIER
 #define ALC_ALL_DEVICES_SPECIFIER 0x1013
 #endif
 
+namespace
+{
+    const int loudnessFPS = 20; // loudness values per second of audio
+}
 
 namespace MWSound
 {
@@ -750,7 +755,7 @@ void OpenAL_Output::deinit()
 }
 
 
-ALuint OpenAL_Output::getBuffer(const std::string &fname)
+ALuint OpenAL_Output::getBuffer(const std::string &fname, std::vector<float>* loudnessBuffer)
 {
     ALuint buf = 0;
 
@@ -765,11 +770,12 @@ ALuint OpenAL_Output::getBuffer(const std::string &fname)
             if(iter != mUnusedBuffers.end())
                 mUnusedBuffers.erase(iter);
         }
-
-        return buf;
     }
     throwALerror();
 
+    if (buf != 0 && loudnessBuffer == NULL)
+        return buf;
+
     std::vector<char> data;
     ChannelConfig chans;
     SampleType type;
@@ -795,42 +801,50 @@ ALuint OpenAL_Output::getBuffer(const std::string &fname)
     decoder->readAll(data);
     decoder->close();
 
-    alGenBuffers(1, &buf);
+    if (loudnessBuffer != NULL)
-    throwALerror();
-
-    alBufferData(buf, format, &data[0], data.size(), srate);
-    mBufferCache[fname] = buf;
-    mBufferRefs[buf] = 1;
-
-    ALint bufsize = 0;
-    alGetBufferi(buf, AL_SIZE, &bufsize);
-    mBufferCacheMemSize += bufsize;
-
-    // NOTE: Max buffer cache: 15MB
-    while(mBufferCacheMemSize > 15*1024*1024)
     {
-        if(mUnusedBuffers.empty())
+        analyzeLoudness(data, srate, chans, type, *loudnessBuffer, loudnessFPS);
+    }
+
+    if (buf == 0)
+    {
+        alGenBuffers(1, &buf);
+        throwALerror();
+
+        alBufferData(buf, format, &data[0], data.size(), srate);
+        mBufferCache[fname] = buf;
+        mBufferRefs[buf] = 1;
+
+        ALint bufsize = 0;
+        alGetBufferi(buf, AL_SIZE, &bufsize);
+        mBufferCacheMemSize += bufsize;
+
+        // NOTE: Max buffer cache: 15MB
+        while(mBufferCacheMemSize > 15*1024*1024)
         {
-            std::cout <<"No more unused buffers to clear!"<< std::endl;
+            if(mUnusedBuffers.empty())
-            break;
+            {
+                std::cout <<"No more unused buffers to clear!"<< std::endl;
+                break;
+            }
+
+            ALuint oldbuf = mUnusedBuffers.front();
+            mUnusedBuffers.pop_front();
+
+            NameMap::iterator nameiter = mBufferCache.begin();
+            while(nameiter != mBufferCache.end())
+            {
+                if(nameiter->second == oldbuf)
+                    mBufferCache.erase(nameiter++);
+                else
+                    ++nameiter;
+            }
+
+            bufsize = 0;
+            alGetBufferi(oldbuf, AL_SIZE, &bufsize);
+            alDeleteBuffers(1, &oldbuf);
+            mBufferCacheMemSize -= bufsize;
         }
-
-        ALuint oldbuf = mUnusedBuffers.front();
-        mUnusedBuffers.pop_front();
-
-        NameMap::iterator nameiter = mBufferCache.begin();
-        while(nameiter != mBufferCache.end())
-        {
-            if(nameiter->second == oldbuf)
-                mBufferCache.erase(nameiter++);
-            else
-                ++nameiter;
-        }
-
-        bufsize = 0;
-        alGetBufferi(oldbuf, AL_SIZE, &bufsize);
-        alDeleteBuffers(1, &oldbuf);
-        mBufferCacheMemSize -= bufsize;
     }
     return buf;
 }
@@ -883,7 +897,7 @@ MWBase::SoundPtr OpenAL_Output::playSound(const std::string &fname, float vol, f
 }
 
 MWBase::SoundPtr OpenAL_Output::playSound3D(const std::string &fname, const Ogre::Vector3 &pos, float vol, float basevol, float pitch,
-                                            float min, float max, int flags, float offset)
+                                            float min, float max, int flags, float offset, bool extractLoudness)
 {
     boost::shared_ptr<OpenAL_Sound> sound;
     ALuint src=0, buf=0;
@@ -895,8 +909,12 @@ MWBase::SoundPtr OpenAL_Output::playSound3D(const std::string &fname, const Ogre
 
     try
     {
-        buf = getBuffer(fname);
+        std::vector<float> loudnessVector;
+
+        buf = getBuffer(fname, extractLoudness ? &loudnessVector : NULL);
+
         sound.reset(new OpenAL_Sound3D(*this, src, buf, pos, vol, basevol, pitch, min, max, flags));
+        sound->setLoudnessVector(loudnessVector, loudnessFPS);
     }
     catch(std::exception&)
     {

apps/openmw/mwsound/openal_output.hpp

@@ -36,7 +36,7 @@ namespace MWSound
         typedef std::vector<Sound*> SoundVec;
         SoundVec mActiveSounds;
 
-        ALuint getBuffer(const std::string &fname);
+        ALuint getBuffer(const std::string &fname, std::vector<float>* loudnessBuffer=NULL);
         void bufferFinished(ALuint buffer);
 
         Environment mLastEnvironment;
@@ -49,7 +49,7 @@ namespace MWSound
         virtual MWBase::SoundPtr playSound(const std::string &fname, float vol, float basevol, float pitch, int flags, float offset);
         /// @param offset Value from [0,1] meaning from which fraction the sound the playback starts.
         virtual MWBase::SoundPtr playSound3D(const std::string &fname, const Ogre::Vector3 &pos,
-                                             float vol, float basevol, float pitch, float min, float max, int flags, float offset);
+                                             float vol, float basevol, float pitch, float min, float max, int flags, float offset, bool extractLoudness=false);
         virtual MWBase::SoundPtr streamSound(DecoderPtr decoder, float volume, float pitch, int flags);
 
         virtual void updateListener(const Ogre::Vector3 &pos, const Ogre::Vector3 &atdir, const Ogre::Vector3 &updir, Environment env);

apps/openmw/mwsound/sound.cpp

@@ -0,0 +1,23 @@
+#include "sound.hpp"
+
+namespace MWSound
+{
+
+    float Sound::getCurrentLoudness()
+    {
+        if (mLoudnessVector.empty())
+            return 0.f;
+        int index = getTimeOffset() * mLoudnessFPS;
+
+        index = std::max(0, std::min(index, int(mLoudnessVector.size()-1)));
+
+        return mLoudnessVector[index];
+    }
+
+    void Sound::setLoudnessVector(const std::vector<float> &loudnessVector, float loudnessFPS)
+    {
+        mLoudnessVector = loudnessVector;
+        mLoudnessFPS = loudnessFPS;
+    }
+
+}

apps/openmw/mwsound/sound.hpp

@@ -24,6 +24,9 @@ namespace MWSound
         int mFlags;
         float mFadeOutTime;
 
+        std::vector<float> mLoudnessVector;
+        float mLoudnessFPS;
+
     public:
         virtual void stop() = 0;
         virtual bool isPlaying() = 0;
@@ -31,6 +34,12 @@ namespace MWSound
         void setPosition(const Ogre::Vector3 &pos) { mPos = pos; }
         void setVolume(float volume) { mVolume = volume; }
         void setFadeout(float duration) { mFadeOutTime=duration; }
+        void setLoudnessVector(const std::vector<float>& loudnessVector, float loudnessFPS);
+
+        /// Get loudness at the current time position on a [0,1] scale.
+        /// Requires that loudnessVector was filled in by the user.
+        float getCurrentLoudness();
+
         MWBase::SoundManager::PlayType getPlayType() const
         { return (MWBase::SoundManager::PlayType)(mFlags&MWBase::SoundManager::Play_TypeMask); }
 
@@ -44,6 +53,7 @@ namespace MWSound
           , mMaxDistance(maxdist)
           , mFlags(flags)
           , mFadeOutTime(0)
+          , mLoudnessFPS(20)
         { }
         virtual ~Sound() { }
 

apps/openmw/mwsound/sound_output.hpp

@@ -28,7 +28,7 @@ namespace MWSound
         virtual MWBase::SoundPtr playSound(const std::string &fname, float vol, float basevol, float pitch, int flags, float offset) = 0;
         /// @param offset Value from [0,1] meaning from which fraction the sound the playback starts.
         virtual MWBase::SoundPtr playSound3D(const std::string &fname, const Ogre::Vector3 &pos,
-                                             float vol, float basevol, float pitch, float min, float max, int flags, float offset) = 0;
+                                             float vol, float basevol, float pitch, float min, float max, int flags, float offset, bool extractLoudness=false) = 0;
         virtual MWBase::SoundPtr streamSound(DecoderPtr decoder, float volume, float pitch, int flags) = 0;
 
         virtual void updateListener(const Ogre::Vector3 &pos, const Ogre::Vector3 &atdir, const Ogre::Vector3 &updir, Environment env) = 0;

apps/openmw/mwsound/soundmanagerimp.cpp

@@ -256,7 +256,7 @@ namespace MWSound
             const Ogre::Vector3 objpos(pos.pos);
 
             MWBase::SoundPtr sound = mOutput->playSound3D(filePath, objpos, 1.0f, basevol, 1.0f,
-                                                          20.0f, 1500.0f, Play_Normal|Play_TypeVoice, 0);
+                                                          20.0f, 1500.0f, Play_Normal|Play_TypeVoice, 0, true);
             mActiveSounds[sound] = std::make_pair(ptr, std::string("_say_sound"));
         }
         catch(std::exception &e)
@@ -265,6 +265,21 @@ namespace MWSound
         }
     }
 
+    float SoundManager::getSaySoundLoudness(const MWWorld::Ptr &ptr) const
+    {
+        SoundMap::const_iterator snditer = mActiveSounds.begin();
+        while(snditer != mActiveSounds.end())
+        {
+            if(snditer->second.first == ptr && snditer->second.second == "_say_sound")
+                break;
+            ++snditer;
+        }
+        if (snditer == mActiveSounds.end())
+            return 0.f;
+
+        return snditer->first->getCurrentLoudness();
+    }
+
     void SoundManager::say(const std::string& filename)
     {
         if(!mOutput->isInitialized())

apps/openmw/mwsound/soundmanagerimp.hpp

@@ -110,6 +110,11 @@ namespace MWSound
         virtual void stopSay(const MWWorld::Ptr &reference=MWWorld::Ptr());
         ///< Stop an actor speaking
 
+        virtual float getSaySoundLoudness(const MWWorld::Ptr& reference) const;
+        ///< Check the currently playing say sound for this actor
+        /// and get an average loudness value (scale [0,1]) at the current time position.
+        /// If the actor is not saying anything, returns 0.
+
         virtual MWBase::SoundPtr playTrack(const DecoderPtr& decoder, PlayType type);
         ///< Play a 2D audio track, using a custom decoder