Help with Pitch and Scrubbing Issues in WaveSurfer

[cold-world]

Hello!

I am currently working with WaveSurfer.js to implement an audio player in my React application. I am facing an issue when trying to preserve the pitch of the audio while changing the playback speed and also enabling scrubbing/seek functionality.

Here is the problem I am encountering:

When using MediaElement backend:
    The pitch remains normal when I adjust the playback speed, which is great.
    However, scrubbing (seeking) does not work properly. I am unable to scrub through the audio effectively when the backend: 'MediaElement' option is used.

When using WebAudio backend:
    Scrubbing and seeking work as expected.
    However, when I adjust the playback speed, the pitch becomes distorted, which makes the voice sound like a "cartoonish" version.

I need to find a way to preserve the normal pitch even when I change the playback speed and keep the scrubbing/seek functionality working as expected.

`import './AudioPlayer.scss';
import { useEffect, useRef, useState } from 'react';
import WaveSurfer from 'wavesurfer.js';

type AudioPlayerProps = {
fileUrl: string;
playIcon: JSX.Element;
pauseIcon: JSX.Element;
variant?: 'meeting' | 'input' | 'message';
audioId: string;
};

export const AudioPlayer = ({
fileUrl,
playIcon,
variant = 'message',
audioId,
pauseIcon,
}: AudioPlayerProps) => {
const containerRef = useRef(null);
const wavesurferRef = useRef<WaveSurfer | null>(null);
const [localDuration, setLocalDuration] = useState('00:00');
const [isWaveformReady, setIsWaveformReady] = useState(false);

const { setCurrentAudio, currentAudioId, isPlaying, playbackSpeed } = useAudioPlayerStore();

useEffect(() => {
if (!containerRef.current) return;

const waveColor = variant === 'meeting' ? '#fff' : '#5A91F1';
const wavesurfer = WaveSurfer.create({
  container: containerRef.current,
  waveColor,
  progressColor: '#387AEF',
  height: 16,
  barWidth: 2,
  barGap: 2,
  barRadius: 0,
  cursorWidth: 0,
  normalize: true,
  backend: 'WebAudio', // WebAudio backend used here
  mediaControls: false,
  interact: true,
  hideScrollbar: true,
});

wavesurferRef.current = wavesurfer;

wavesurfer.on('ready', () => {
  const duration = wavesurfer.getDuration();
  setLocalDuration(formatDuration(duration));
  setIsWaveformReady(true);
  wavesurfer.setPlaybackRate(playbackSpeed);
});

wavesurfer.load(fileUrl);

return () => {
  if (currentAudioId === audioId) {
    useAudioPlayerStore.getState().clearCurrentAudio();
  }
  wavesurfer.destroy();
};

}, [fileUrl]);

useEffect(() => {
if (wavesurferRef.current && currentAudioId === audioId) {
wavesurferRef.current.setPlaybackRate(playbackSpeed);
}
}, [playbackSpeed]);

const handlePlayPause = () => {
if (!isWaveformReady || !wavesurferRef.current) return;
if (currentAudioId !== audioId) {
setCurrentAudio(audioId, variant, wavesurferRef.current);
} else {
useAudioPlayerStore.getState().togglePlayPause();
}
};

return (

{isPlaying && currentAudioId === audioId ? pauseIcon : playIcon}

{localDuration}

);
};
`

[katspaugh]

You could use WebAudio + a custom pitch shifting effect like in this example:

https://wavesurfer.xyz/examples/?phase-vocoder/index.js

(Source code: https://github.com/katspaugh/wavesurfer.js/tree/main/examples/phase-vocoder)

[cold-world]

@katspaugh Thanks for answer.

Now pitch preservation works correctly with plugin, but seeking through the waveform by clicking no longer works. I've tried both 'WebAudio' and 'MediaElement' backends, but neither preserves the seeking functionality.
How can I implement pitch preservation while maintaining the ability to seek through the waveform? Is there a way to properly connect the AudioWorklet without breaking wavesurfer's seeking functionality?
Any help would be greatly appreciated.
Thanks!

`export const AudioPlayer = ({
fileUrl,
playIcon,
variant = 'message',
audioId,
pauseIcon,
}: AudioPlayerProps) => {
const containerRef = useRef(null);
const wavesurferRef = useRef<WaveSurfer | null>(null);
const audioElementRef = useRef<HTMLAudioElement | null>(null);
const phaseVocoderNodeRef = useRef<AudioWorkletNode | null>(null);
const audioContextRef = useRef<AudioContext | null>(null);

const [localDuration, setLocalDuration] = useState('00:00');
const [isWaveformReady, setIsWaveformReady] = useState(false);

const { setCurrentAudio, currentAudioId, isPlaying, playbackSpeed } =
useAudioPlayerStore();

useEffect(() => {
if (!containerRef.current) return;

const audioElement = document.createElement('audio');
audioElement.src = fileUrl;
audioElement.crossOrigin = 'anonymous';
audioElementRef.current = audioElement;

const waveColor = variant === 'meeting' ? '#fff' : '#5A91F1';
const wavesurfer = WaveSurfer.create({
  container: containerRef.current,
  waveColor,
  progressColor: '#387AEF',
  height: 16,
  barWidth: 2,
  barGap: 2,
  barRadius: 0,
  cursorWidth: 1,
  normalize: true,
  barAlign: variant === 'input' ? undefined : 'bottom',
  backend: 'WebAudio',
  media: audioElement,
  hideScrollbar: true,
  interact: true,
  url: fileUrl,
});

wavesurferRef.current = wavesurfer;

const setupPitchPreservation = async () => {
  const audioContext = new window.AudioContext();
  audioContextRef.current = audioContext;

  try {
    await audioContext.audioWorklet.addModule('/phase-vocoder.js');

    const phaseVocoderNode = new AudioWorkletNode(
      audioContext,
      'phase-vocoder-processor',
    );
    phaseVocoderNodeRef.current = phaseVocoderNode;

    const sourceNode = audioContext.createMediaElementSource(audioElement);

    const gainNode = audioContext.createGain();

    sourceNode.connect(gainNode);
    gainNode.connect(phaseVocoderNode);
    phaseVocoderNode.connect(audioContext.destination);

    const updatePitch = (speed: number) => {
      if (phaseVocoderNode) {
        const pitchFactorParam =
          phaseVocoderNode.parameters.get('pitchFactor');
        if (pitchFactorParam) {
          const adjustedFactor = 1 - (1 - 1 / speed) * 0.1;
          pitchFactorParam.value = adjustedFactor;
        }
      }
    };

    wavesurfer.on('ready', () => {
      const duration = wavesurfer.getDuration();
      setLocalDuration(formatDuration(duration));
      setIsWaveformReady(true);

      wavesurfer.setPlaybackRate(playbackSpeed);
      updatePitch(playbackSpeed);
    });

    (wavesurferRef.current as any).updatePitch = updatePitch;
  } catch (error) {
    console.error('Error setting up pitch preservation:', error);
    setIsWaveformReady(true);
  }
};

setupPitchPreservation();

wavesurfer.on('finish', () => {
  useAudioPlayerStore.getState().clearCurrentAudio();
});

wavesurfer.on('pause', () => {
  if (currentAudioId === audioId) {
    useAudioPlayerStore.getState().togglePlayPause();
  }
});

wavesurfer.on('play', () => {
  if (currentAudioId === audioId && !isPlaying) {
    useAudioPlayerStore.getState().togglePlayPause();
  }
});

return () => {
  if (currentAudioId === audioId) {
    useAudioPlayerStore.getState().clearCurrentAudio();
  }

  if (phaseVocoderNodeRef.current) {
    phaseVocoderNodeRef.current.disconnect();
  }

  if (audioContextRef.current) {
    audioContextRef.current.close();
  }

  wavesurfer.destroy();
  audioElement.remove();
};

}, [fileUrl]);

useEffect(() => {
const wavesurfer = wavesurferRef.current;
const updatePitch = wavesurfer && (wavesurfer as any).updatePitch;

if (wavesurfer && currentAudioId === audioId) {
  wavesurfer.setPlaybackRate(playbackSpeed);

  if (updatePitch) {
    updatePitch(playbackSpeed);
  }
}

}, [playbackSpeed, currentAudioId]);
`