Add C++ runtime for models from 3d-speaker

.github/scripts/test-speaker-recognition-python.sh

@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+
+set -e
+
+log() {
+  # This function is from espnet
+  local fname=${BASH_SOURCE[1]##*/}
+  echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*"
+}
+
+d=/tmp/sr-models
+mkdir -p $d
+
+pushd $d
+log "Download test waves"
+git clone https://github.com/csukuangfj/sr-data
+popd
+
+log "Download wespeaker models"
+model_dir=$d/wespeaker
+mkdir -p $model_dir
+pushd $model_dir
+models=(
+en_voxceleb_CAM++.onnx
+en_voxceleb_CAM++_LM.onnx
+en_voxceleb_resnet152_LM.onnx
+en_voxceleb_resnet221_LM.onnx
+en_voxceleb_resnet293_LM.onnx
+en_voxceleb_resnet34.onnx
+en_voxceleb_resnet34_LM.onnx
+zh_cnceleb_resnet34.onnx
+zh_cnceleb_resnet34_LM.onnx
+)
+for m in ${models[@]}; do
+  wget -q https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-recongition-models/$m
+done
+ls -lh
+popd
+
+log "Download 3d-speaker models"
+model_dir=$d/3dspeaker
+mkdir -p $model_dir
+pushd $model_dir
+models=(
+speech_campplus_sv_en_voxceleb_16k.onnx
+speech_campplus_sv_zh-cn_16k-common.onnx
+speech_eres2net_base_200k_sv_zh-cn_16k-common.onnx
+speech_eres2net_base_sv_zh-cn_3dspeaker_16k.onnx
+speech_eres2net_large_sv_zh-cn_3dspeaker_16k.onnx
+speech_eres2net_sv_en_voxceleb_16k.onnx
+speech_eres2net_sv_zh-cn_16k-common.onnx
+)
+for m in ${models[@]}; do
+  wget -q https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-recongition-models/$m
+done
+ls -lh
+popd
+
+
+python3 sherpa-onnx/python/tests/test_speaker_recognition.py --verbose

.github/workflows/run-python-test.yaml

@@ -76,6 +76,7 @@ jobs:
       - name: Test sherpa-onnx
         shell: bash
         run: |
+          .github/scripts/test-speaker-recognition-python.sh
           .github/scripts/test-python.sh
 
       - uses: actions/upload-artifact@v3

sherpa-onnx/csrc/CMakeLists.txt

@@ -99,7 +99,7 @@ set(sources
 # speaker embedding extractor
 list(APPEND sources
   speaker-embedding-extractor-impl.cc
-  speaker-embedding-extractor-wespeaker-model.cc
+  speaker-embedding-extractor-model.cc
   speaker-embedding-extractor.cc
   speaker-embedding-manager.cc
 )

sherpa-onnx/csrc/speaker-embedding-extractor-wespeaker-impl.h → sherpa-onnx/csrc/speaker-embedding-extractor-general-impl.h

@@ -1,31 +1,32 @@
-// sherpa-onnx/csrc/speaker-embedding-extractor-wespeaker-impl.h
+// sherpa-onnx/csrc/speaker-embedding-extractor-general-impl.h
 //
 // Copyright (c)  2023  Xiaomi Corporation
 
-#ifndef SHERPA_ONNX_CSRC_SPEAKER_EMBEDDING_EXTRACTOR_WESPEAKER_IMPL_H_
-#define SHERPA_ONNX_CSRC_SPEAKER_EMBEDDING_EXTRACTOR_WESPEAKER_IMPL_H_
+#ifndef SHERPA_ONNX_CSRC_SPEAKER_EMBEDDING_EXTRACTOR_GENERAL_IMPL_H_
+#define SHERPA_ONNX_CSRC_SPEAKER_EMBEDDING_EXTRACTOR_GENERAL_IMPL_H_
 #include <algorithm>
 #include <memory>
 #include <utility>
 #include <vector>
 
+#include "Eigen/Dense"
 #include "sherpa-onnx/csrc/speaker-embedding-extractor-impl.h"
-#include "sherpa-onnx/csrc/speaker-embedding-extractor-wespeaker-model.h"
+#include "sherpa-onnx/csrc/speaker-embedding-extractor-model.h"
 
 namespace sherpa_onnx {
 
-class SpeakerEmbeddingExtractorWeSpeakerImpl
+class SpeakerEmbeddingExtractorGeneralImpl
     : public SpeakerEmbeddingExtractorImpl {
  public:
-  explicit SpeakerEmbeddingExtractorWeSpeakerImpl(
+  explicit SpeakerEmbeddingExtractorGeneralImpl(
       const SpeakerEmbeddingExtractorConfig &config)
       : model_(config) {}
 
   int32_t Dim() const override { return model_.GetMetaData().output_dim; }
 
   std::unique_ptr<OnlineStream> CreateStream() const override {
     FeatureExtractorConfig feat_config;
-    auto meta_data = model_.GetMetaData();
+    const auto &meta_data = model_.GetMetaData();
     feat_config.sampling_rate = meta_data.sample_rate;
     feat_config.normalize_samples = meta_data.normalize_samples;
 
@@ -52,6 +53,17 @@ class SpeakerEmbeddingExtractorWeSpeakerImpl
 
     int32_t feat_dim = features.size() / num_frames;
 
+    const auto &meta_data = model_.GetMetaData();
+    if (!meta_data.feature_normalize_type.empty()) {
+      if (meta_data.feature_normalize_type == "global-mean") {
+        SubtractGlobalMean(features.data(), num_frames, feat_dim);
+      } else {
+        SHERPA_ONNX_LOGE("Unsupported feature_normalize_type: %s",
+                         meta_data.feature_normalize_type.c_str());
+        exit(-1);
+      }
+    }
+
     auto memory_info =
         Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeDefault);
 
@@ -71,9 +83,19 @@ class SpeakerEmbeddingExtractorWeSpeakerImpl
   }
 
  private:
-  SpeakerEmbeddingExtractorWeSpeakerModel model_;
+  void SubtractGlobalMean(float *p, int32_t num_frames,
+                          int32_t feat_dim) const {
+    auto m = Eigen::Map<
+        Eigen::Matrix<float, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor>>(
+        p, num_frames, feat_dim);
+
+    m = m.rowwise() - m.colwise().mean();
+  }
+
+ private:
+  SpeakerEmbeddingExtractorModel model_;
 };
 
 }  // namespace sherpa_onnx
 
-#endif  // SHERPA_ONNX_CSRC_SPEAKER_EMBEDDING_EXTRACTOR_WESPEAKER_IMPL_H_
+#endif  // SHERPA_ONNX_CSRC_SPEAKER_EMBEDDING_EXTRACTOR_GENERAL_IMPL_H_

sherpa-onnx/csrc/speaker-embedding-extractor-impl.cc

@@ -5,14 +5,15 @@
 
 #include "sherpa-onnx/csrc/macros.h"
 #include "sherpa-onnx/csrc/onnx-utils.h"
-#include "sherpa-onnx/csrc/speaker-embedding-extractor-wespeaker-impl.h"
+#include "sherpa-onnx/csrc/speaker-embedding-extractor-general-impl.h"
 
 namespace sherpa_onnx {
 
 namespace {
 
 enum class ModelType {
   kWeSpeaker,
+  k3dSpeaker,
   kUnkown,
 };
 
@@ -49,6 +50,8 @@ static ModelType GetModelType(char *model_data, size_t model_data_length,
 
   if (model_type.get() == std::string("wespeaker")) {
     return ModelType::kWeSpeaker;
+  } else if (model_type.get() == std::string("3d-speaker")) {
+    return ModelType::k3dSpeaker;
   } else {
     SHERPA_ONNX_LOGE("Unsupported model_type: %s", model_type.get());
     return ModelType::kUnkown;
@@ -68,7 +71,9 @@ SpeakerEmbeddingExtractorImpl::Create(
 
   switch (model_type) {
     case ModelType::kWeSpeaker:
-      return std::make_unique<SpeakerEmbeddingExtractorWeSpeakerImpl>(config);
+      // fall through
+    case ModelType::k3dSpeaker:
+      return std::make_unique<SpeakerEmbeddingExtractorGeneralImpl>(config);
     case ModelType::kUnkown:
       SHERPA_ONNX_LOGE(
           "Unknown model type in for speaker embedding extractor!");

sherpa-onnx/csrc/speaker-embedding-extractor-model-meta-data.h

@@ -0,0 +1,28 @@
+// sherpa-onnx/csrc/speaker-embedding-extractor-model-meta-data.h
+//
+// Copyright (c)  2023  Xiaomi Corporation
+#ifndef SHERPA_ONNX_CSRC_SPEAKER_EMBEDDING_EXTRACTOR_MODEL_META_DATA_H_
+#define SHERPA_ONNX_CSRC_SPEAKER_EMBEDDING_EXTRACTOR_MODEL_META_DATA_H_
+
+#include <cstdint>
+#include <string>
+
+namespace sherpa_onnx {
+
+struct SpeakerEmbeddingExtractorModelMetaData {
+  int32_t output_dim = 0;
+  int32_t sample_rate = 0;
+
+  // for wespeaker models, it is 0;
+  // for 3d-speaker models, it is 1
+  int32_t normalize_samples = 1;
+
+  // Chinese, English, etc.
+  std::string language;
+
+  // for 3d-speaker, it is global-mean
+  std::string feature_normalize_type;
+};
+
+}  // namespace sherpa_onnx
+#endif  // SHERPA_ONNX_CSRC_SPEAKER_EMBEDDING_EXTRACTOR_MODEL_META_DATA_H_

sherpa-onnx/csrc/speaker-embedding-extractor-wespeaker-model.cc → sherpa-onnx/csrc/speaker-embedding-extractor-model.cc

@@ -1,8 +1,8 @@
-// sherpa-onnx/csrc/speaker-embedding-extractor-wespeaker-model.cc
+// sherpa-onnx/csrc/speaker-embedding-extractor-model.cc
 //
-// Copyright (c)  2023  Xiaomi Corporation
+// Copyright (c)  2023-2024  Xiaomi Corporation
 
-#include "sherpa-onnx/csrc/speaker-embedding-extractor-wespeaker-model.h"
+#include "sherpa-onnx/csrc/speaker-embedding-extractor-model.h"
 
 #include <string>
 #include <utility>
@@ -11,11 +11,11 @@
 #include "sherpa-onnx/csrc/macros.h"
 #include "sherpa-onnx/csrc/onnx-utils.h"
 #include "sherpa-onnx/csrc/session.h"
-#include "sherpa-onnx/csrc/speaker-embedding-extractor-wespeaker-model-metadata.h"
+#include "sherpa-onnx/csrc/speaker-embedding-extractor-model-meta-data.h"
 
 namespace sherpa_onnx {
 
-class SpeakerEmbeddingExtractorWeSpeakerModel::Impl {
+class SpeakerEmbeddingExtractorModel::Impl {
  public:
   explicit Impl(const SpeakerEmbeddingExtractorConfig &config)
       : config_(config),
@@ -37,7 +37,7 @@ class SpeakerEmbeddingExtractorWeSpeakerModel::Impl {
     return std::move(outputs[0]);
   }
 
-  const SpeakerEmbeddingExtractorWeSpeakerModelMetaData &GetMetaData() const {
+  const SpeakerEmbeddingExtractorModelMetaData &GetMetaData() const {
     return meta_data_;
   }
 
@@ -65,10 +65,13 @@ class SpeakerEmbeddingExtractorWeSpeakerModel::Impl {
                                "normalize_samples");
     SHERPA_ONNX_READ_META_DATA_STR(meta_data_.language, "language");
 
+    SHERPA_ONNX_READ_META_DATA_STR_WITH_DEFAULT(
+        meta_data_.feature_normalize_type, "feature_normalize_type", "");
+
     std::string framework;
     SHERPA_ONNX_READ_META_DATA_STR(framework, "framework");
-    if (framework != "wespeaker") {
-      SHERPA_ONNX_LOGE("Expect a wespeaker model, given: %s",
+    if (framework != "wespeaker" && framework != "3d-speaker") {
+      SHERPA_ONNX_LOGE("Expect a wespeaker or a 3d-speaker model, given: %s",
                        framework.c_str());
       exit(-1);
     }
@@ -88,24 +91,21 @@ class SpeakerEmbeddingExtractorWeSpeakerModel::Impl {
   std::vector<std::string> output_names_;
   std::vector<const char *> output_names_ptr_;
 
-  SpeakerEmbeddingExtractorWeSpeakerModelMetaData meta_data_;
+  SpeakerEmbeddingExtractorModelMetaData meta_data_;
 };
 
-SpeakerEmbeddingExtractorWeSpeakerModel::
-    SpeakerEmbeddingExtractorWeSpeakerModel(
-        const SpeakerEmbeddingExtractorConfig &config)
+SpeakerEmbeddingExtractorModel::SpeakerEmbeddingExtractorModel(
+    const SpeakerEmbeddingExtractorConfig &config)
     : impl_(std::make_unique<Impl>(config)) {}
 
-SpeakerEmbeddingExtractorWeSpeakerModel::
-    ~SpeakerEmbeddingExtractorWeSpeakerModel() = default;
+SpeakerEmbeddingExtractorModel::~SpeakerEmbeddingExtractorModel() = default;
 
-const SpeakerEmbeddingExtractorWeSpeakerModelMetaData &
-SpeakerEmbeddingExtractorWeSpeakerModel::GetMetaData() const {
+const SpeakerEmbeddingExtractorModelMetaData &
+SpeakerEmbeddingExtractorModel::GetMetaData() const {
   return impl_->GetMetaData();
 }
 
-Ort::Value SpeakerEmbeddingExtractorWeSpeakerModel::Compute(
-    Ort::Value x) const {
+Ort::Value SpeakerEmbeddingExtractorModel::Compute(Ort::Value x) const {
   return impl_->Compute(std::move(x));
 }
 

sherpa-onnx/csrc/speaker-embedding-extractor-model.h

@@ -0,0 +1,37 @@
+// sherpa-onnx/csrc/speaker-embedding-extractor-model.h
+//
+// Copyright (c)  2023-2024  Xiaomi Corporation
+#ifndef SHERPA_ONNX_CSRC_SPEAKER_EMBEDDING_EXTRACTOR_MODEL_H_
+#define SHERPA_ONNX_CSRC_SPEAKER_EMBEDDING_EXTRACTOR_MODEL_H_
+
+#include <memory>
+
+#include "onnxruntime_cxx_api.h"  // NOLINT
+#include "sherpa-onnx/csrc/speaker-embedding-extractor-model-meta-data.h"
+#include "sherpa-onnx/csrc/speaker-embedding-extractor.h"
+
+namespace sherpa_onnx {
+
+class SpeakerEmbeddingExtractorModel {
+ public:
+  explicit SpeakerEmbeddingExtractorModel(
+      const SpeakerEmbeddingExtractorConfig &config);
+
+  ~SpeakerEmbeddingExtractorModel();
+
+  const SpeakerEmbeddingExtractorModelMetaData &GetMetaData() const;
+
+  /**
+   * @param x A float32 tensor of shape (N, T, C)
+   * @return A float32 tensor of shape (N, C)
+   */
+  Ort::Value Compute(Ort::Value x) const;
+
+ private:
+  class Impl;
+  std::unique_ptr<Impl> impl_;
+};
+
+}  // namespace sherpa_onnx
+
+#endif  // SHERPA_ONNX_CSRC_SPEAKER_EMBEDDING_EXTRACTOR_MODEL_H_