add verbose option

Signed-off-by: Kentaro Hayashi <kenhys@gmail.com>
Gbp-Pq: Name 0011-add-verbose-option.patch

diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml
index 7f190836f46c17c39a4c999f4e0f88ce9b9583db..510807418988b412d6c8f8e57fe1a7c890c392f6 100644 (file)
--- a/.github/workflows/cmake.yml
+++ b/.github/workflows/cmake.yml
@@ -45,7 +45,7 @@ jobs:
 
     - name: Test
       working-directory: ${{github.workspace}}/build
-      run: ctest -C Release
+      run: ctest -C Release --output-on-failure
 
     - name: Package
       working-directory: ${{github.workspace}}/build

diff --git a/src/common.h b/src/common.h
index c27c3522e18093930aa600e954cf7d755c8546af..ba951d6b066c60d614c3d05b28baea885ce4829d 100644 (file)
--- a/src/common.h
+++ b/src/common.h
@@ -98,15 +98,6 @@ class Die {
  private:
   bool die_;
 };
-
-template <typename T>
-T &&CheckNotNull(const char *file, int line, const char *exprtext, T &&t) {
-  if (t == nullptr) {
-    std::cerr << file << "(" << line << ") " << exprtext;
-    Abort();
-  }
-  return std::forward<T>(t);
-}
 }  // namespace error
 
 namespace logging {
@@ -158,10 +149,6 @@ inline const char *BaseName(const char *path) {
 #define CHECK_LE(a, b) CHECK((a) <= (b))
 #define CHECK_GT(a, b) CHECK((a) > (b))
 #define CHECK_LT(a, b) CHECK((a) < (b))
-#define CHECK_NOTNULL(val)                                    \
-  ::sentencepiece::error::CheckNotNull(                       \
-      ::sentencepiece::logging::BaseName(__FILE__), __LINE__, \
-      "'" #val "' Must be non NULL", (val))
 
 #define FRIEND_TEST(a, b) friend class a##_Test_##b;
 

diff --git a/src/normalizer.cc b/src/normalizer.cc
index d87f89b3b1706a66b7514a2e3300e797322536b8..2ab808495874d669dd58edffff0eb819fd947993 100644 (file)
--- a/src/normalizer.cc
+++ b/src/normalizer.cc
@@ -12,11 +12,12 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.!
 
+#include "normalizer.h"
+
 #include <utility>
 #include <vector>
 
 #include "common.h"
-#include "normalizer.h"
 #include "third_party/absl/memory/memory.h"
 #include "third_party/absl/strings/match.h"
 #include "third_party/absl/strings/string_view.h"
@@ -46,9 +47,7 @@ Normalizer::~Normalizer() {}
 
 void Normalizer::Init() {
   absl::string_view index = spec_->precompiled_charsmap();
-  if (index.empty()) {
-    LOG(INFO) << "precompiled_charsmap is empty. use identity normalization.";
-  } else {
+  if (!index.empty()) {
     absl::string_view trie_blob, normalized;
 #ifdef IS_BIG_ENDIAN
     status_ = DecodePrecompiledCharsMap(index, &trie_blob, &normalized,

diff --git a/src/sentencepiece_processor.cc b/src/sentencepiece_processor.cc
index a6f53953a3de9ca27f002c9c701e2624e0d902d8..805e0f9335482af1fad58c922a2670d9d51d1b94 100644 (file)
--- a/src/sentencepiece_processor.cc
+++ b/src/sentencepiece_processor.cc
@@ -67,12 +67,12 @@ ImmutableSentencePieceText::ImmutableSentencePiece::ImmutableSentencePiece(
     const SentencePieceText_SentencePiece &sp)
     : sp_(&sp) {}
 
-absl::string_view ImmutableSentencePieceText::ImmutableSentencePiece::piece()
+const std::string &ImmutableSentencePieceText::ImmutableSentencePiece::piece()
     const {
   return sp_->piece();
 }
 
-absl::string_view ImmutableSentencePieceText::ImmutableSentencePiece::surface()
+const std::string &ImmutableSentencePieceText::ImmutableSentencePiece::surface()
     const {
   return sp_->surface();
 }
@@ -109,8 +109,10 @@ ImmutableSentencePieceText::pieces(int index) const {
       spt_->pieces(index));
 }
 
-absl::string_view ImmutableSentencePieceText::text() const {
-  return spt_ ? spt_->text() : "";
+const std::string &ImmutableSentencePieceText::text() const {
+  if (spt_) return spt_->text();
+  static std::string *kEmptyString = new std::string();
+  return *kEmptyString;
 }
 
 float ImmutableSentencePieceText::score() const {

diff --git a/src/sentencepiece_processor.h b/src/sentencepiece_processor.h
index 51c5b3bac7ea33bbbacdcc1ed89ef70e2b4b7eb4..8124c5923591eae7523e9caccd44c3264364fdfd 100644 (file)
--- a/src/sentencepiece_processor.h
+++ b/src/sentencepiece_processor.h
@@ -165,8 +165,8 @@ class ImmutableSentencePieceText {
   class ImmutableSentencePiece {
    public:
     ~ImmutableSentencePiece() = default;
-    absl::string_view piece() const;
-    absl::string_view surface() const;
+    const std::string &piece() const;
+    const std::string &surface() const;
     uint32_t id() const;
     uint32_t begin() const;
     uint32_t end() const;
@@ -182,7 +182,7 @@ class ImmutableSentencePieceText {
   std::vector<ImmutableSentencePiece> pieces() const;
   size_t pieces_size() const;
   ImmutableSentencePiece pieces(int index) const;
-  absl::string_view text() const;
+  const std::string &text() const;
   float score() const;
 
   std::string SerializeAsString() const;
@@ -193,7 +193,6 @@ class ImmutableSentencePieceText {
   SentencePieceText *mutable_proto();
 
   friend class ImmutableNBestSentencePieceText;
-  friend class SentencePieceProcessor;
 
  private:
   explicit ImmutableSentencePieceText(const SentencePieceText &spt);
@@ -222,8 +221,6 @@ class ImmutableNBestSentencePieceText {
   // it returns the raw pointer managed by the shared_ptr.
   NBestSentencePieceText *mutable_proto();
 
-  friend class SentencePieceProcessor;
-
  private:
   std::shared_ptr<NBestSentencePieceText> rep_;
 };

diff --git a/src/util.h b/src/util.h
index fb312f100982f8efd0458c58b8dedce473cdf9f2..01a561f777c80cdc48f35d37ed33fa54e63580ed 100644 (file)
--- a/src/util.h
+++ b/src/util.h
@@ -94,7 +94,6 @@ inline bool lexical_cast(absl::string_view arg, std::string *result) {
 
 template <typename T>
 inline bool DecodePOD(absl::string_view str, T *result) {
-  CHECK_NOTNULL(result);
   if (sizeof(*result) != str.size()) {
     return false;
   }