%include exception.i
%{
+#include <iostream>
#include <algorithm>
+#include <functional>
#include <limits>
#include <cmath>
+#include <thread>
+#include <vector>
#include <sentencepiece_processor.h>
#include <sentencepiece_trainer.h>
PyObject* kUnicodeInput = reinterpret_cast<PyObject* >(0x1);
PyObject* kByteInput = reinterpret_cast<PyObject* >(0x2);
+using BytesArray = std::vector<sentencepiece::util::bytes>;
+
inline void ReleaseResultObject(PyObject *obj) {
if (obj != nullptr && obj != kUnicodeInput && obj != kByteInput) {
Py_XDECREF(obj);
return PyBytes_FromStringAndSize(output.data(), output.size());
}
-PyObject* MakePyOutputBytes(const std::string& output) {
+PyObject* MakePyOutputBytes(const sentencepiece::util::bytes& output) {
return PyBytes_FromStringAndSize(output.data(), output.size());
}
sentencepiece::util::Status status_;
};
-void RewriteIds(const sentencepiece::SentencePieceProcessor &sp,
- std::vector<int> *ids,
- bool add_bos, bool add_eos, bool reverse) {
+inline void RewriteIds(const sentencepiece::SentencePieceProcessor &sp,
+ std::vector<int> *ids,
+ bool add_bos, bool add_eos, bool reverse, bool emit_unk_piece) {
if (!add_bos && !add_eos && !reverse) return;
if (reverse) std::reverse(ids->begin(), ids->end());
if (add_bos) ids->insert(ids->begin(), sp.bos_id());
if (add_eos) ids->push_back(sp.eos_id());
}
-void RewritePieces(const sentencepiece::SentencePieceProcessor &sp,
- std::vector<std::string> *pieces,
- bool add_bos, bool add_eos, bool reverse, bool emit_unk_piece) {
+inline void RewriteIds(const sentencepiece::SentencePieceProcessor &sp,
+ std::vector<std::string> *pieces,
+ bool add_bos, bool add_eos, bool reverse, bool emit_unk_piece) {
if (!add_bos && !add_eos && !reverse && !emit_unk_piece) return;
if (reverse) std::reverse(pieces->begin(), pieces->end());
if (add_bos) pieces->insert(pieces->begin(), sp.IdToPiece(sp.bos_id()));
}
}
}
+
+inline void RewriteIds(const sentencepiece::SentencePieceProcessor &sp,
+ sentencepiece::util::bytes *proto,
+ bool add_bos, bool add_eos, bool reverse, bool emit_unk_piece) {
+ if (add_bos || add_eos || reverse || emit_unk_piece) {
+ throw sentencepiece::util::Status(
+ sentencepiece::util::StatusCode::kUnimplemented,
+ "add_bos, add_eos, reverse, and emit_unk_piece is not supported in AsSerialize API");
+ }
+}
+
+inline void CheckIds(const std::vector<int> &ids, int num_pieces) {
+ for (int id : ids) {
+ if (id < 0 || id >= num_pieces) {
+ throw sentencepiece::util::Status(
+ sentencepiece::util::StatusCode::kOutOfRange,
+ "piece id is out of range.");
+ }
+ }
+}
+
+inline void CheckIds(const std::vector<std::string> &ids, int num_pieces) {}
+
+class ThreadPool {
+ public:
+ explicit ThreadPool(size_t request_size) :
+ request_size_(request_size) {}
+
+ virtual ~ThreadPool() {
+ for (auto &task : tasks_) {
+ task.join();
+ }
+ }
+
+ void Schedule(std::function<void()> closure) {
+ static constexpr size_t kMinThreadSize = 2;
+ if (request_size_ < kMinThreadSize) {
+ closure();
+ } else {
+ tasks_.emplace_back(closure);
+ }
+ }
+
+ private:
+ size_t request_size_ = 0;
+ std::vector<std::thread> tasks_;
+};
+
+template <typename T>
+inline void InitNumThreads(const std::vector<T> &ins, int *num_threads) {
+ *num_threads = std::max<int>(1,
+ std::min<int>({*num_threads,
+ static_cast<int>(ins.size()), 256}));
+}
+
+#define DEFINE_ENCODE_BATCH_FUNC_IMPL(FuncName, InType, OutType) \
+ std::vector<OutType> outs(ins.size()); \
+ InitNumThreads(ins, &num_threads); \
+ { \
+ ThreadPool pool(ins.size()); \
+ for (int n = 0; n < num_threads; ++n) { \
+ pool.Schedule([&, n]() { \
+ for (size_t i = n; i < ins.size(); i += num_threads) { \
+ auto out = enable_sampling ? \
+ self->Sample##FuncName(ins[i], \
+ nbest_size, alpha) : \
+ self->FuncName(ins[i]); \
+ RewriteIds(*self, &out, add_bos, add_eos, reverse, \
+ emit_unk_piece); \
+ outs[i] = std::move(out); \
+ } \
+ }); \
+ } \
+ } \
+ return outs;
+
+#define DEFINE_DECODE_BATCH_FUNC_IMPL(FuncName, InType, OutType) \
+ std::vector<OutType> outs(ins.size()); \
+ InitNumThreads(ins, &num_threads); \
+ { \
+ ThreadPool pool(ins.size()); \
+ for (int n = 0; n < num_threads; ++n) { \
+ pool.Schedule([&, n]() { \
+ for (size_t i = n; i < ins.size(); i += num_threads) { \
+ CheckIds(ins[i], self->GetPieceSize()); \
+ outs[i] = self->FuncName(ins[i]); \
+ } \
+ }); \
+ } \
+ } \
+ return outs;
+
} // namespace
%}
%ignore sentencepiece::SentencePieceText;
%ignore sentencepiece::NormalizerSpec;
%ignore sentencepiece::TrainerSpec;
-
%ignore sentencepiece::SentencePieceProcessor::status;
+
%ignore sentencepiece::SentencePieceProcessor::Encode;
+%ignore sentencepiece::SentencePieceProcessor::EncodeAsPieces;
+%ignore sentencepiece::SentencePieceProcessor::EncodeAsIds;
+%ignore sentencepiece::SentencePieceProcessor::EncodeAsSerializedProto;
%ignore sentencepiece::SentencePieceProcessor::SampleEncode;
+%ignore sentencepiece::SentencePieceProcessor::SampleEncodeAsIds;
+%ignore sentencepiece::SentencePieceProcessor::SampleEncodeAsPieces;
+%ignore sentencepiece::SentencePieceProcessor::SampleEncodeAsSerializedProto;
%ignore sentencepiece::SentencePieceProcessor::NBestEncode;
+%ignore sentencepiece::SentencePieceProcessor::NBestEncodeAsPieces;
+%ignore sentencepiece::SentencePieceProcessor::NBestEncodeAsIds;
+%ignore sentencepiece::SentencePieceProcessor::NBestEncodeAsSerializedProto;
%ignore sentencepiece::SentencePieceProcessor::SampleEncodeAndScore;
+
%ignore sentencepiece::SentencePieceProcessor::Decode;
%ignore sentencepiece::SentencePieceProcessor::DecodeIds;
+%ignore sentencepiece::SentencePieceProcessor::DecodePieces;
+%ignore sentencepiece::SentencePieceProcessor::DecodePiecesAsSerializedProto;
%ignore sentencepiece::SentencePieceProcessor::DecodeIdsAsSerializedProto;
+
%ignore sentencepiece::SentencePieceProcessor::model_proto;
%ignore sentencepiece::SentencePieceProcessor::Load;
%ignore sentencepiece::SentencePieceProcessor::LoadOrDie;
return $self->Load(arg);
}
- std::string DecodeIdsWithCheck(
- const std::vector<int> &ids) const {
- const int num_pieces = $self->GetPieceSize();
- for (int id : ids) {
- if (id < 0 || id >= num_pieces) {
- throw sentencepiece::util::Status(
- sentencepiece::util::StatusCode::kOutOfRange,
- "piece id is out of range.");
- }
- }
- return $self->DecodeIds(ids);
- }
-
- util::bytes DecodeIdsAsSerializedProtoWithCheck(
- const std::vector<int> &ids) const {
- const int num_pieces = $self->GetPieceSize();
- for (int id : ids) {
- if (id < 0 || id >= num_pieces) {
- throw sentencepiece::util::Status(
- sentencepiece::util::StatusCode::kOutOfRange,
- "piece id is out of range.");
- }
- }
- return $self->DecodeIdsAsSerializedProto(ids);
- }
-
+ /////////////////////////////////////////////////////////////////////////////
+ // EncodeAs* (Single request)
std::vector<int> _EncodeAsIds(absl::string_view text,
- bool enabele_sampling,
+ bool enable_sampling,
int nbest_size, float alpha,
- bool add_bos, bool add_eos, bool reverse) {
- auto ids = enabele_sampling ?
+ bool add_bos, bool add_eos, bool reverse,
+ bool emit_unk_piece) const {
+ auto ids = enable_sampling ?
$self->SampleEncodeAsIds(text, nbest_size, alpha) :
$self->EncodeAsIds(text);
- RewriteIds(*$self, &ids, add_bos, add_eos, reverse);
+ RewriteIds(*$self, &ids, add_bos, add_eos, reverse, emit_unk_piece);
return ids;
}
std::vector<std::string> _EncodeAsPieces(absl::string_view text,
- bool enabele_sampling,
+ bool enable_sampling,
int nbest_size, float alpha,
bool add_bos, bool add_eos, bool reverse,
- bool emit_unk_piece) {
- auto pieces = enabele_sampling ?
+ bool emit_unk_piece) const {
+ auto pieces = enable_sampling ?
$self->SampleEncodeAsPieces(text, nbest_size, alpha) :
$self->EncodeAsPieces(text);
- RewritePieces(*$self, &pieces, add_bos, add_eos, reverse, emit_unk_piece);
+ RewriteIds(*$self, &pieces, add_bos, add_eos, reverse, emit_unk_piece);
return pieces;
}
+ sentencepiece::util::bytes _EncodeAsSerializedProto(absl::string_view text,
+ bool enable_sampling,
+ int nbest_size, float alpha,
+ bool add_bos, bool add_eos, bool reverse,
+ bool emit_unk_piece) const {
+ auto proto = enable_sampling ?
+ $self->SampleEncodeAsSerializedProto(text, nbest_size, alpha) :
+ $self->EncodeAsSerializedProto(text);
+ RewriteIds(*$self, &proto, add_bos, add_eos, reverse, emit_unk_piece);
+ return proto;
+ }
+
+ /////////////////////////////////////////////////////////////////////////////
+ // EncodeAs* (Batch request)
+ std::vector<std::vector<int>> _EncodeAsIdsBatch(
+ const std::vector<absl::string_view> &ins, int num_threads,
+ bool enable_sampling, int nbest_size, float alpha,
+ bool add_bos, bool add_eos, bool reverse,
+ bool emit_unk_piece) const {
+ DEFINE_ENCODE_BATCH_FUNC_IMPL(EncodeAsIds,
+ absl::string_view, std::vector<int>);
+ }
+
+ std::vector<std::vector<std::string>> _EncodeAsPiecesBatch(
+ const std::vector<absl::string_view> &ins, int num_threads,
+ bool enable_sampling, int nbest_size, float alpha,
+ bool add_bos, bool add_eos, bool reverse,
+ bool emit_unk_piece) const {
+ DEFINE_ENCODE_BATCH_FUNC_IMPL(EncodeAsPieces,
+ absl::string_view, std::vector<std::string>);
+ }
+
+ BytesArray _EncodeAsSerializedProtoBatch(
+ const std::vector<absl::string_view> &ins, int num_threads,
+ bool enable_sampling, int nbest_size, float alpha,
+ bool add_bos, bool add_eos, bool reverse,
+ bool emit_unk_piece) const {
+ DEFINE_ENCODE_BATCH_FUNC_IMPL(EncodeAsSerializedProto,
+ absl::string_view,
+ sentencepiece::util::bytes);
+ }
+
+ /////////////////////////////////////////////////////////////////////////////
+ // DecodeAs* (Single request)
+ std::string _DecodeIds(const std::vector<int> &ids) const {
+ CheckIds(ids, $self->GetPieceSize());
+ return $self->DecodeIds(ids);
+ }
+
+ std::string _DecodePieces(const std::vector<std::string> &pieces) const {
+ return $self->DecodePieces(pieces);
+ }
+
+ sentencepiece::util::bytes _DecodeIdsAsSerializedProto(
+ const std::vector<int> &ids) const {
+ CheckIds(ids, $self->GetPieceSize());
+ return $self->DecodeIdsAsSerializedProto(ids);
+ }
+
+ sentencepiece::util::bytes _DecodePiecesAsSerializedProto(
+ const std::vector<std::string> &pieces) const {
+ CheckIds(pieces, $self->GetPieceSize());
+ return $self->DecodePiecesAsSerializedProto(pieces);
+ }
+
+ /////////////////////////////////////////////////////////////////////////////
+ // DecodeAs* (Batch request)
+ std::vector<std::string> _DecodeIdsBatch(
+ const std::vector<std::vector<int>> &ins, int num_threads) const {
+ DEFINE_DECODE_BATCH_FUNC_IMPL(DecodeIds, int, std::string);
+ }
+
+ BytesArray _DecodeIdsAsSerializedProtoBatch(
+ const std::vector<std::vector<int>> &ins, int num_threads) const {
+ DEFINE_DECODE_BATCH_FUNC_IMPL(DecodeIdsAsSerializedProto, int,
+ sentencepiece::util::bytes);
+ }
+
+ std::vector<std::string> _DecodePiecesBatch(
+ const std::vector<std::vector<std::string>> &ins, int num_threads) const {
+ DEFINE_DECODE_BATCH_FUNC_IMPL(DecodePieces, std::string, std::string);
+ }
+
+ BytesArray _DecodePiecesAsSerializedProtoBatch(
+ const std::vector<std::vector<std::string>> &ins, int num_threads) const {
+ DEFINE_DECODE_BATCH_FUNC_IMPL(DecodePiecesAsSerializedProto, std::string,
+ sentencepiece::util::bytes);
+ }
+
+ ////////////////////////////////////////////////////////////////////////////
+ // NBestEncodeAs* (Single request)
std::vector<std::vector<int>>
_NBestEncodeAsIds(absl::string_view text,
int nbest_size,
- bool add_bos, bool add_eos, bool reverse) {
+ bool add_bos, bool add_eos, bool reverse,
+ bool emit_unk_piece) const {
auto idss = $self->NBestEncodeAsIds(text, nbest_size);
for (auto &ids : idss) {
- RewriteIds(*$self, &ids, add_bos, add_eos, reverse);
+ RewriteIds(*$self, &ids, add_bos, add_eos, reverse, emit_unk_piece);
}
return idss;
}
_NBestEncodeAsPieces(absl::string_view text,
int nbest_size,
bool add_bos, bool add_eos, bool reverse,
- bool emit_unk_piece) {
+ bool emit_unk_piece) const {
auto piecess = $self->NBestEncodeAsPieces(text, nbest_size);
for (auto &pieces : piecess) {
- RewritePieces(*$self, &pieces, add_bos, add_eos, reverse, emit_unk_piece);
+ RewriteIds(*$self, &pieces, add_bos, add_eos, reverse, emit_unk_piece);
}
return piecess;
}
+ sentencepiece::util::bytes _NBestEncodeAsSerializedProto(absl::string_view text,
+ int nbest_size,
+ bool add_bos, bool add_eos, bool reverse,
+ bool emit_unk_piece) const {
+ RewriteIds(*$self, static_cast<sentencepiece::util::bytes *>(nullptr),
+ add_bos, add_eos, reverse, emit_unk_piece);
+ return $self->NBestEncodeAsSerializedProto(text, nbest_size);
+ }
+
+ /////////////////////////////////////////////////////////////////////////////
+ // SampleEncodeAndScoreAs* (Single request)
std::vector<std::pair<std::vector<int>, float>>
_SampleEncodeAndScoreAsIds(absl::string_view text,
int num_samples, float theta, bool wor,
bool include_best,
- bool add_bos, bool add_eos, bool reverse) {
+ bool add_bos, bool add_eos, bool reverse,
+ bool emit_unk_piece) const {
auto idss = $self->SampleEncodeAndScoreAsIds(text, num_samples,
theta, wor, include_best);
for (auto &ids : idss) {
- RewriteIds(*$self, &ids.first, add_bos, add_eos, reverse);
+ RewriteIds(*$self, &ids.first, add_bos, add_eos, reverse, emit_unk_piece);
}
return idss;
}
- std::vector<std::pair<std::vector<std::string>, float>>
+ std::vector<std::pair<std::vector<std::string>, float>>
_SampleEncodeAndScoreAsPieces(absl::string_view text,
int num_samples, float theta, bool wor,
bool include_best,
bool add_bos, bool add_eos, bool reverse,
- bool emit_unk_piece) {
+ bool emit_unk_piece) const {
auto piecess = $self->SampleEncodeAndScoreAsPieces(text, num_samples,
theta, wor, include_best);
for (auto &pieces : piecess) {
- RewritePieces(*$self, &pieces.first, add_bos, add_eos, reverse, emit_unk_piece);
+ RewriteIds(*$self, &pieces.first, add_bos, add_eos, reverse, emit_unk_piece);
}
return piecess;
- }
+ }
+
+ // Calculate Entropy
+ float _CalculateEntropy(absl::string_view text, float theta) {
+ return $self->CalculateEntropy(text, theta);
+ }
+
+ std::vector<float> _CalculateEntropyBatch(const std::vector<absl::string_view> &ins,
+ float theta, int num_threads) {
+ std::vector<float> outs(ins.size());
+ InitNumThreads(ins, &num_threads);
+ {
+ ThreadPool pool(ins.size());
+ for (int n = 0; n < num_threads; ++n) {
+ pool.Schedule([&, n]() {
+ for (size_t i = n; i < ins.size(); i += num_threads) {
+ outs[i] = self->CalculateEntropy(ins[i], theta);
+ }
+ });
+ }
+ }
+ return outs;
+ }
%pythoncode {
def Init(self,
emit_unk_piece=False,
enable_sampling=False,
nbest_size=-1,
- alpha=0.1):
+ alpha=0.1,
+ num_threads=1):
"""Initialzie sentencepieceProcessor.
Args:
forward-filtering-and-backward-sampling algorithm.
alpha: Soothing parameter for unigram sampling, and dropout probability of
merge operations for BPE-dropout.
+ num_threads: number of threads in batch processing.
"""
_sentencepiece_processor_init_native(self)
self._enable_sampling = enable_sampling
self._nbest_size = nbest_size
self._alpha = alpha
+ self._num_threads = num_threads
if model_file or model_proto:
self.Load(model_file=model_file, model_proto=model_proto)
emit_unk_piece=None,
enable_sampling=None,
nbest_size=None,
- alpha=None):
+ alpha=None,
+ num_threads=None):
"""Encode text input to segmented ids or tokens.
Args:
forward-filtering-and-backward-sampling algorithm.
alpha: Soothing parameter for unigram sampling, and merge probability for
BPE-dropout (probablity 'p' in BPE-dropout paper).
+ num_threads: the number of threads used in the batch processin (Default = 1).
"""
if out_type is None:
nbest_size = self._nbest_size
if alpha is None:
alpha = self._alpha
+ if num_threads is None:
+ num_threads = self._num_threads
if enable_sampling == True and (nbest_size is None or nbest_size == 0 or
nbest_size == 1 or alpha is None):
'instead of nbest segmentations.'
)
- def _encode(text):
- if out_type is int:
- return self._EncodeAsIds(text, enable_sampling, nbest_size,
- alpha, add_bos, add_eos, reverse)
- else:
- return self._EncodeAsPieces(text, enable_sampling, nbest_size,
- alpha, add_bos, add_eos, reverse, emit_unk_piece)
+ if num_threads is None or type(num_threads) is not int:
+ raise RuntimeError('num_threads must be int')
if type(input) is list:
- return [_encode(n) for n in input]
+ if out_type is int:
+ return self._EncodeAsIdsBatch(input, num_threads, enable_sampling, nbest_size,
+ alpha, add_bos, add_eos, reverse, emit_unk_piece)
+ if out_type is str:
+ return self._EncodeAsPiecesBatch(input, num_threads, enable_sampling, nbest_size,
+ alpha, add_bos, add_eos, reverse, emit_unk_piece)
+ if out_type == 'proto':
+ return self._EncodeAsSerializedProtoBatch(input, num_threads, enable_sampling, nbest_size,
+ alpha, add_bos, add_eos, reverse, emit_unk_piece)
+
+ if out_type is int:
+ return self._EncodeAsIds(input, enable_sampling, nbest_size,
+ alpha, add_bos, add_eos, reverse, emit_unk_piece)
+ if out_type is str:
+ return self._EncodeAsPieces(input, enable_sampling, nbest_size,
+ alpha, add_bos, add_eos, reverse, emit_unk_piece)
+ if out_type == 'proto':
+ return self._EncodeAsSerializedProto(input, enable_sampling, nbest_size,
+ alpha, add_bos, add_eos, reverse, emit_unk_piece)
+
+ raise RuntimeError('unknown out_type={}'.format(out_type))
+ return None
- return _encode(input)
+
+ def EncodeAsPieces(self, input, **kwargs):
+ return self.Encode(input=input, out_type=str, **kwargs)
+
+
+ def EncodeAsIds(self, input, **kwargs):
+ return self.Encode(input=input, out_type=int, **kwargs)
+
+
+ def EncodeAsSerializedProto(self, input, **kwargs):
+ return self.Encode(input=input, out_type='proto', **kwargs)
+
+
+ def SampleEncodeAsPieces(self, input, nbest_size=None, alpha=None, **kwargs):
+ return self.Encode(input=input, nbest_size=nbest_size, alpha=alpha,
+ out_type=str, enable_sampling=True, **kwargs)
+
+
+ def SampleEncodeAsIds(self, input, nbest_size=None, alpha=None,**kwargs):
+ return self.Encode(input=input, nbest_size=nbest_size, alpha=alpha,
+ out_type=int, enable_sampling=True, **kwargs)
+
+
+ def SampleEncodeAsSerializedProto(self, input, nbest_size=None, alpha=None, **kwargs):
+ return self.Encode(input=input, nbest_size=nbest_size, alpha=alpha,
+ out_type='proto', enable_sampling=True, **kwargs)
def NBestEncode(self,
def _encode(text):
if out_type is int:
- return self._NBestEncodeAsIds(text, nbest_size, add_bos, add_eos, reverse)
- else:
- return self._NBestEncodeAsPieces(text, nbest_size, add_bos, add_eos, reverse, emit_unk_piece)
+ return self._NBestEncodeAsIds(text, nbest_size,
+ add_bos, add_eos, reverse, emit_unk_piece)
+ if out_type is str:
+ return self._NBestEncodeAsPieces(text, nbest_size,
+ add_bos, add_eos, reverse, emit_unk_piece)
+ if out_type == 'proto':
+ return self._NBestEncodeAsSerializedProto(text, nbest_size,
+ add_bos, add_eos, reverse, emit_unk_piece)
if type(input) is list:
return [_encode(n) for n in input]
return _encode(input)
+ def NBestEncodeAsPieces(self, input, nbest_size=None, **kwargs):
+ return self.NBestEncode(input=input, nbest_size=nbest_size,
+ out_type=str, **kwargs)
+
+
+ def NBestEncodeAsIds(self, input, nbest_size=None, **kwargs):
+ return self.NBestEncode(input=input, nbest_size=nbest_size,
+ out_type=int, **kwargs)
+
+
+ def NBestEncodeAsSerializedProto(self, input, nbest_size=None, **kwargs):
+ return self.NBestEncode(input=input, nbest_size=nbest_size,
+ out_type='proto', **kwargs)
+
+
def SampleEncodeAndScore(self,
input,
out_type=None,
Args:
input: input string. accepsts list of string.
- out_type: output type. int or str.
+ out_type: output type. int or str or 'proto'.
add_bos: Add <s> to the result (Default = false)
add_eos: Add </s> to the result (Default = false) <s>/</s> is added after reversing (if enabled).
reverse: Reverses the tokenized sequence (Default = false)
if include_best and not wor:
raise RuntimeError('When include_best is True, We must specify "wor = True".')
-
+
def _encode(text):
if out_type is int:
return self._SampleEncodeAndScoreAsIds(text, num_samples, theta, wor, include_best,
- add_bos, add_eos, reverse)
+ add_bos, add_eos, reverse, emit_unk_piece)
else:
return self._SampleEncodeAndScoreAsPieces(text, num_samples, theta, wor, include_best,
add_bos, add_eos, reverse, emit_unk_piece)
return _encode(input)
- def Decode(self, input):
- """Decode processed id or token sequences."""
+ def Decode(self, input, out_type=str, num_threads=None):
+ """Decode processed id or token sequences.
+
+ Args:
+ out_type: output type. str or 'proto' (Default = str)
+ num_threads: the number of threads used in the batch processin (Default = 1).
+ """
+
+ if num_threads is None:
+ num_threads = self._num_threads
+
+ if num_threads is None or type(num_threads) is not int:
+ raise RuntimeError('num_threads must be int')
if not input:
- return self.DecodeIds([])
- elif type(input) is int:
- return self.DecodeIdsWithCheck([input])
- elif type(input) is str:
- return self.DecodePieces([input])
+ return ''
+
+ if out_type is str:
+ if type(input) is int:
+ return self._DecodeIds([input])
+ if type(input) is str:
+ return self._DecodePieces([input])
+
+ if type(input) is list:
+ if len(input) == 0 or type(input[0]) is int:
+ return self._DecodeIds(input)
+ if type(input[0]) is str:
+ return self._DecodePieces(input)
+
+ if type(input[0]) is list:
+ if len(input[0]) == 0 or type(input[0][0]) is int:
+ return self._DecodeIdsBatch(input, num_threads)
+ if type(input[0][0]) is str:
+ return self._DecodePiecesBatch(input, num_threads)
+
+ if out_type == 'proto':
+ if type(input) is int:
+ return self._DecodeIdsAsSerializedProto([input])
+ if type(input) is str:
+ return self._DecodePiecesAsSerializedProto([input])
+
+ if type(input) is list:
+ if len(input) == 0 or type(input[0]) is int:
+ return self._DecodeIdsAsSerializedProto(input)
+ if type(input[0]) is str:
+ return self._DecodePiecesAsSerializedProto(input)
+
+ if type(input[0]) is list:
+ if len(input[0]) == 0 or type(input[0][0]) is int:
+ return self._DecodeIdsAsSerializedProtoBatch(input, num_threads)
+ if type(input[0][0]) is str:
+ return self._DecodePiecesAsSerializedProtoBatch(input, num_threads)
+
+
+ raise RuntimeError('unknown output or input type')
+ return None
- def _decode(input):
- if not input:
- return self.DecodeIds([])
- if type(input[0]) is int:
- return self.DecodeIdsWithCheck(input)
- return self.DecodePieces(input)
- if type(input[0]) is list:
- return [_decode(n) for n in input]
+ def DecodePieces(self, input, out_type=str, **kwargs):
+ return self.Decode(input=input, out_type=out_type, **kwargs)
- return _decode(input)
+ def DecodeIds(self, input, out_type=str, **kwargs):
+ return self.Decode(input=input, out_type=out_type, **kwargs)
+
+
+ def DecodePiecesAsSerializedProto(self, input, out_type='proto', **kwargs):
+ return self.Decode(input=input, out_type=out_type, **kwargs)
- def Entropy(self, input, theta):
- """Calculate sentence entropy"""
+ def DecodeIdsAsSerializedProto(self, input, out_type='proto', **kwargs):
+ return self.Decode(input=input, out_type=out_type, **kwargs)
+
+
+ def CalculateEntropy(self, input, theta, num_threads=None):
+ """Calculate sentence entropy"""
if type(input) is list:
- return [self.CalculateEntropy(n, theta) for n in input]
- return self.CalculateEntropy(input, theta)
+ if num_threads is None:
+ num_threads = self._num_threads
+ if num_threads is None or type(num_threads) is not int:
+ raise RuntimeError('num_threads must be int')
+ return self._CalculateEntropyBatch(input, theta, num_threads)
+
+ return self._CalculateEntropy(input, theta)
def piece_size(self):
}
}
+%typemap(out) std::vector<float> {
+ $result = PyList_New($1.size());
+ for (size_t i = 0; i < $1.size(); ++i) {
+ PyList_SetItem($result, i, PyFloat_FromDouble(static_cast<double>($1[i])));
+ }
+}
+
%typemap(out) std::vector<std::vector<int>> {
$result = PyList_New($1.size());
for (size_t i = 0; i < $1.size(); ++i) {
}
}
+%typemap(out) BytesArray {
+ $result = PyList_New($1.size());
+ for (size_t i = 0; i < $1.size(); ++i) {
+ PyList_SetItem($result, i, MakePyOutputBytes($1[i]));
+ }
+}
+
%typemap(out) std::vector<std::vector<std::string>> {
PyObject *input_type = resultobj;
$result = PyList_New($1.size());
for (size_t i = 0; i < size; ++i) {
const PyInputString ustring(PyList_GetItem($input, i));
if (ustring.IsAvalable()) {
- (*out)[i] = std::string(ustring.data(), ustring.size());
+ (*out)[i].assign(ustring.data(), ustring.size());
+ } else {
+ PyErr_SetString(PyExc_TypeError, "list must contain strings");
+ SWIG_fail;
+ }
+ resultobj = ustring.input_type();
+ }
+ } else {
+ PyErr_SetString(PyExc_TypeError, "not a list");
+ SWIG_fail;
+ }
+ $1 = out;
+}
+
+%typemap(in) const std::vector<absl::string_view>& {
+ std::vector<absl::string_view> *out = nullptr;
+ if (PyList_Check($input)) {
+ const size_t size = PyList_Size($input);
+ out = new std::vector<std::string>(size);
+ for (size_t i = 0; i < size; ++i) {
+ const PyInputString ustring(PyList_GetItem($input, i));
+ if (ustring.IsAvalable()) {
+ (*out)[i] = absl::string_view(ustring.data(), ustring.size());
+ } else {
+ PyErr_SetString(PyExc_TypeError, "list must contain strings");
+ SWIG_fail;
+ }
+ resultobj = ustring.input_type();
+ }
+ } else {
+ PyErr_SetString(PyExc_TypeError, "not a list");
+ SWIG_fail;
+ }
+ $1 = out;
+}
+
+%typemap(in) const std::vector<absl::string_view>& {
+ std::vector<absl::string_view> *out = nullptr;
+ if (PyList_Check($input)) {
+ const size_t size = PyList_Size($input);
+ out = new std::vector<absl::string_view>(size);
+ for (size_t i = 0; i < size; ++i) {
+ const PyInputString ustring(PyList_GetItem($input, i));
+ if (ustring.IsAvalable()) {
+ (*out)[i] = absl::string_view(ustring.data(), ustring.size());
} else {
PyErr_SetString(PyExc_TypeError, "list must contain strings");
SWIG_fail;
$1 = out;
}
+%typemap(in) const std::vector<std::vector<std::string>>& {
+ std::vector<std::vector<std::string>> *out = nullptr;
+ if (PyList_Check($input)) {
+ const size_t size = PyList_Size($input);
+ out = new std::vector<std::vector<std::string>>(size);
+ for (size_t i = 0; i < size; ++i) {
+ PyObject *o = PyList_GetItem($input, i);
+ if (PyList_Check(o)) {
+ const size_t size2 = PyList_Size(o);
+ (*out)[i].resize(size2);
+ for (size_t j = 0; j < size2; ++j) {
+ const PyInputString ustring(PyList_GetItem(o, j));
+ if (ustring.IsAvalable()) {
+ (*out)[i][j].assign(ustring.data(), ustring.size());
+ } else {
+ PyErr_SetString(PyExc_TypeError,"list must contain integers");
+ SWIG_fail;
+ }
+ resultobj = ustring.input_type();
+ }
+ } else {
+ PyErr_SetString(PyExc_TypeError,"not a list");
+ SWIG_fail;
+ }
+ }
+ } else {
+ PyErr_SetString(PyExc_TypeError,"not a list");
+ SWIG_fail;
+ }
+ $1 = out;
+}
+
+%typemap(in) const std::vector<std::vector<int>>& {
+ std::vector<std::vector<int>> *out = nullptr;
+ if (PyList_Check($input)) {
+ const size_t size = PyList_Size($input);
+ out = new std::vector<std::vector<int>>(size);
+ for (size_t i = 0; i < size; ++i) {
+ PyObject *o = PyList_GetItem($input, i);
+ if (PyList_Check(o)) {
+ const size_t size2 = PyList_Size(o);
+ (*out)[i].resize(size2);
+ for (size_t j = 0; j < size2; ++j) {
+ PyObject *o2 = PyList_GetItem(o, j);
+ if (PyInt_Check(o2)) {
+ (*out)[i][j] = static_cast<int>(PyInt_AsLong(o2));
+ } else {
+ PyErr_SetString(PyExc_TypeError, "list must contain strings");
+ SWIG_fail;
+ }
+ }
+ } else {
+ PyErr_SetString(PyExc_TypeError, "not a list");
+ SWIG_fail;
+ }
+ }
+ } else {
+ PyErr_SetString(PyExc_TypeError,"not a list");
+ SWIG_fail;
+ }
+ $1 = out;
+}
+
%typemap(in) const std::unordered_map<std::string, std::string> & {
std::unordered_map<std::string, std::string> *out = nullptr;
if (PyDict_Check($input)) {
delete $1;
}
+%typemap(freearg) const std::vector<absl::string_view>& {
+ delete $1;
+}
+
%typemap(freearg) const std::vector<std::vector<std::string>>& {
delete $1;
}
delete $1;
}
+%typemap(freearg) const std::vector<float>& {
+ delete $1;
+}
+
%typemap(freearg) const std::vector<std::vector<int>>& {
delete $1;
}
SentencePieceProcessor.Tokenize = SentencePieceProcessor.Encode
SentencePieceProcessor.Detokenize = SentencePieceProcessor.Decode
-SentencePieceProcessor.DecodeIds = SentencePieceProcessor.DecodeIdsWithCheck
-SentencePieceProcessor.DecodeIdsAsSerializedProto = SentencePieceProcessor.DecodeIdsAsSerializedProtoWithCheck
for m in [
'PieceToId', 'IdToPiece', 'GetScore', 'IsUnknown', 'IsControl', 'IsUnused',
#define SWIGTYPE_p_sentencepiece__SentencePieceTrainer swig_types[3]
#define SWIGTYPE_p_std__string swig_types[4]
#define SWIGTYPE_p_std__unordered_mapT_std__string_std__string_t swig_types[5]
-#define SWIGTYPE_p_std__vectorT_int_t swig_types[6]
-#define SWIGTYPE_p_std__vectorT_std__string_t swig_types[7]
-static swig_type_info *swig_types[9];
-static swig_module_info swig_module = {swig_types, 8, 0, 0, 0, 0};
+#define SWIGTYPE_p_std__vectorT_absl__string_view_t swig_types[6]
+#define SWIGTYPE_p_std__vectorT_int_t swig_types[7]
+#define SWIGTYPE_p_std__vectorT_std__string_t swig_types[8]
+#define SWIGTYPE_p_std__vectorT_std__vectorT_int_t_t swig_types[9]
+#define SWIGTYPE_p_std__vectorT_std__vectorT_std__string_t_t swig_types[10]
+static swig_type_info *swig_types[12];
+static swig_module_info swig_module = {swig_types, 11, 0, 0, 0, 0};
#define SWIG_TypeQuery(name) SWIG_TypeQueryModule(&swig_module, &swig_module, name)
#define SWIG_MangledTypeQuery(name) SWIG_MangledTypeQueryModule(&swig_module, &swig_module, name)
}
+#include <iostream>
#include <algorithm>
+#include <functional>
#include <limits>
#include <cmath>
+#include <thread>
+#include <vector>
#include <sentencepiece_processor.h>
#include <sentencepiece_trainer.h>
PyObject* kUnicodeInput = reinterpret_cast<PyObject* >(0x1);
PyObject* kByteInput = reinterpret_cast<PyObject* >(0x2);
+using BytesArray = std::vector<sentencepiece::util::bytes>;
+
inline void ReleaseResultObject(PyObject *obj) {
if (obj != nullptr && obj != kUnicodeInput && obj != kByteInput) {
Py_XDECREF(obj);
return PyBytes_FromStringAndSize(output.data(), output.size());
}
-PyObject* MakePyOutputBytes(const std::string& output) {
+PyObject* MakePyOutputBytes(const sentencepiece::util::bytes& output) {
return PyBytes_FromStringAndSize(output.data(), output.size());
}
sentencepiece::util::Status status_;
};
-void RewriteIds(const sentencepiece::SentencePieceProcessor &sp,
- std::vector<int> *ids,
- bool add_bos, bool add_eos, bool reverse) {
+inline void RewriteIds(const sentencepiece::SentencePieceProcessor &sp,
+ std::vector<int> *ids,
+ bool add_bos, bool add_eos, bool reverse, bool emit_unk_piece) {
if (!add_bos && !add_eos && !reverse) return;
if (reverse) std::reverse(ids->begin(), ids->end());
if (add_bos) ids->insert(ids->begin(), sp.bos_id());
if (add_eos) ids->push_back(sp.eos_id());
}
-void RewritePieces(const sentencepiece::SentencePieceProcessor &sp,
- std::vector<std::string> *pieces,
- bool add_bos, bool add_eos, bool reverse, bool emit_unk_piece) {
+inline void RewriteIds(const sentencepiece::SentencePieceProcessor &sp,
+ std::vector<std::string> *pieces,
+ bool add_bos, bool add_eos, bool reverse, bool emit_unk_piece) {
if (!add_bos && !add_eos && !reverse && !emit_unk_piece) return;
if (reverse) std::reverse(pieces->begin(), pieces->end());
if (add_bos) pieces->insert(pieces->begin(), sp.IdToPiece(sp.bos_id()));
}
}
}
+
+inline void RewriteIds(const sentencepiece::SentencePieceProcessor &sp,
+ sentencepiece::util::bytes *proto,
+ bool add_bos, bool add_eos, bool reverse, bool emit_unk_piece) {
+ if (add_bos || add_eos || reverse || emit_unk_piece) {
+ throw sentencepiece::util::Status(
+ sentencepiece::util::StatusCode::kUnimplemented,
+ "add_bos, add_eos, reverse, and emit_unk_piece is not supported in AsSerialize API");
+ }
+}
+
+inline void CheckIds(const std::vector<int> &ids, int num_pieces) {
+ for (int id : ids) {
+ if (id < 0 || id >= num_pieces) {
+ throw sentencepiece::util::Status(
+ sentencepiece::util::StatusCode::kOutOfRange,
+ "piece id is out of range.");
+ }
+ }
+}
+
+inline void CheckIds(const std::vector<std::string> &ids, int num_pieces) {}
+
+class ThreadPool {
+ public:
+ explicit ThreadPool(size_t request_size) :
+ request_size_(request_size) {}
+
+ virtual ~ThreadPool() {
+ for (auto &task : tasks_) {
+ task.join();
+ }
+ }
+
+ void Schedule(std::function<void()> closure) {
+ static constexpr size_t kMinThreadSize = 2;
+ if (request_size_ < kMinThreadSize) {
+ closure();
+ } else {
+ tasks_.emplace_back(closure);
+ }
+ }
+
+ private:
+ size_t request_size_ = 0;
+ std::vector<std::thread> tasks_;
+};
+
+template <typename T>
+inline void InitNumThreads(const std::vector<T> &ins, int *num_threads) {
+ *num_threads = std::max<int>(1,
+ std::min<int>({*num_threads,
+ static_cast<int>(ins.size()), 256}));
+}
+
+#define DEFINE_ENCODE_BATCH_FUNC_IMPL(FuncName, InType, OutType) \
+ std::vector<OutType> outs(ins.size()); \
+ InitNumThreads(ins, &num_threads); \
+ { \
+ ThreadPool pool(ins.size()); \
+ for (int n = 0; n < num_threads; ++n) { \
+ pool.Schedule([&, n]() { \
+ for (size_t i = n; i < ins.size(); i += num_threads) { \
+ auto out = enable_sampling ? \
+ self->Sample##FuncName(ins[i], \
+ nbest_size, alpha) : \
+ self->FuncName(ins[i]); \
+ RewriteIds(*self, &out, add_bos, add_eos, reverse, \
+ emit_unk_piece); \
+ outs[i] = std::move(out); \
+ } \
+ }); \
+ } \
+ } \
+ return outs;
+
+#define DEFINE_DECODE_BATCH_FUNC_IMPL(FuncName, InType, OutType) \
+ std::vector<OutType> outs(ins.size()); \
+ InitNumThreads(ins, &num_threads); \
+ { \
+ ThreadPool pool(ins.size()); \
+ for (int n = 0; n < num_threads; ++n) { \
+ pool.Schedule([&, n]() { \
+ for (size_t i = n; i < ins.size(); i += num_threads) { \
+ CheckIds(ins[i], self->GetPieceSize()); \
+ outs[i] = self->FuncName(ins[i]); \
+ } \
+ }); \
+ } \
+ } \
+ return outs;
+
} // namespace
SWIGINTERN sentencepiece::util::Status sentencepiece_SentencePieceProcessor_LoadFromFile(sentencepiece::SentencePieceProcessor *self,absl::string_view arg){
return self->Load(arg);
}
-SWIGINTERN std::string sentencepiece_SentencePieceProcessor_DecodeIdsWithCheck(sentencepiece::SentencePieceProcessor const *self,std::vector< int > const &ids){
- const int num_pieces = self->GetPieceSize();
- for (int id : ids) {
- if (id < 0 || id >= num_pieces) {
- throw sentencepiece::util::Status(
- sentencepiece::util::StatusCode::kOutOfRange,
- "piece id is out of range.");
- }
- }
- return self->DecodeIds(ids);
- }
-SWIGINTERN sentencepiece::util::bytes sentencepiece_SentencePieceProcessor_DecodeIdsAsSerializedProtoWithCheck(sentencepiece::SentencePieceProcessor const *self,std::vector< int > const &ids){
- const int num_pieces = self->GetPieceSize();
- for (int id : ids) {
- if (id < 0 || id >= num_pieces) {
- throw sentencepiece::util::Status(
- sentencepiece::util::StatusCode::kOutOfRange,
- "piece id is out of range.");
- }
- }
- return self->DecodeIdsAsSerializedProto(ids);
- }
-SWIGINTERN std::vector< int > sentencepiece_SentencePieceProcessor__EncodeAsIds(sentencepiece::SentencePieceProcessor *self,absl::string_view text,bool enabele_sampling,int nbest_size,float alpha,bool add_bos,bool add_eos,bool reverse){
- auto ids = enabele_sampling ?
+SWIGINTERN std::vector< int > sentencepiece_SentencePieceProcessor__EncodeAsIds(sentencepiece::SentencePieceProcessor const *self,absl::string_view text,bool enable_sampling,int nbest_size,float alpha,bool add_bos,bool add_eos,bool reverse,bool emit_unk_piece){
+ auto ids = enable_sampling ?
self->SampleEncodeAsIds(text, nbest_size, alpha) :
self->EncodeAsIds(text);
- RewriteIds(*self, &ids, add_bos, add_eos, reverse);
+ RewriteIds(*self, &ids, add_bos, add_eos, reverse, emit_unk_piece);
return ids;
}
-SWIGINTERN std::vector< std::string > sentencepiece_SentencePieceProcessor__EncodeAsPieces(sentencepiece::SentencePieceProcessor *self,absl::string_view text,bool enabele_sampling,int nbest_size,float alpha,bool add_bos,bool add_eos,bool reverse,bool emit_unk_piece){
- auto pieces = enabele_sampling ?
+SWIGINTERN std::vector< std::string > sentencepiece_SentencePieceProcessor__EncodeAsPieces(sentencepiece::SentencePieceProcessor const *self,absl::string_view text,bool enable_sampling,int nbest_size,float alpha,bool add_bos,bool add_eos,bool reverse,bool emit_unk_piece){
+ auto pieces = enable_sampling ?
self->SampleEncodeAsPieces(text, nbest_size, alpha) :
self->EncodeAsPieces(text);
- RewritePieces(*self, &pieces, add_bos, add_eos, reverse, emit_unk_piece);
+ RewriteIds(*self, &pieces, add_bos, add_eos, reverse, emit_unk_piece);
return pieces;
}
-SWIGINTERN std::vector< std::vector< int > > sentencepiece_SentencePieceProcessor__NBestEncodeAsIds(sentencepiece::SentencePieceProcessor *self,absl::string_view text,int nbest_size,bool add_bos,bool add_eos,bool reverse){
+SWIGINTERN sentencepiece::util::bytes sentencepiece_SentencePieceProcessor__EncodeAsSerializedProto(sentencepiece::SentencePieceProcessor const *self,absl::string_view text,bool enable_sampling,int nbest_size,float alpha,bool add_bos,bool add_eos,bool reverse,bool emit_unk_piece){
+ auto proto = enable_sampling ?
+ self->SampleEncodeAsSerializedProto(text, nbest_size, alpha) :
+ self->EncodeAsSerializedProto(text);
+ RewriteIds(*self, &proto, add_bos, add_eos, reverse, emit_unk_piece);
+ return proto;
+ }
+SWIGINTERN std::vector< std::vector< int > > sentencepiece_SentencePieceProcessor__EncodeAsIdsBatch(sentencepiece::SentencePieceProcessor const *self,std::vector< absl::string_view > const &ins,int num_threads,bool enable_sampling,int nbest_size,float alpha,bool add_bos,bool add_eos,bool reverse,bool emit_unk_piece){
+ DEFINE_ENCODE_BATCH_FUNC_IMPL(EncodeAsIds,
+ absl::string_view, std::vector<int>);
+ }
+SWIGINTERN std::vector< std::vector< std::string > > sentencepiece_SentencePieceProcessor__EncodeAsPiecesBatch(sentencepiece::SentencePieceProcessor const *self,std::vector< absl::string_view > const &ins,int num_threads,bool enable_sampling,int nbest_size,float alpha,bool add_bos,bool add_eos,bool reverse,bool emit_unk_piece){
+ DEFINE_ENCODE_BATCH_FUNC_IMPL(EncodeAsPieces,
+ absl::string_view, std::vector<std::string>);
+ }
+SWIGINTERN BytesArray sentencepiece_SentencePieceProcessor__EncodeAsSerializedProtoBatch(sentencepiece::SentencePieceProcessor const *self,std::vector< absl::string_view > const &ins,int num_threads,bool enable_sampling,int nbest_size,float alpha,bool add_bos,bool add_eos,bool reverse,bool emit_unk_piece){
+ DEFINE_ENCODE_BATCH_FUNC_IMPL(EncodeAsSerializedProto,
+ absl::string_view,
+ sentencepiece::util::bytes);
+ }
+SWIGINTERN std::string sentencepiece_SentencePieceProcessor__DecodeIds(sentencepiece::SentencePieceProcessor const *self,std::vector< int > const &ids){
+ CheckIds(ids, self->GetPieceSize());
+ return self->DecodeIds(ids);
+ }
+SWIGINTERN std::string sentencepiece_SentencePieceProcessor__DecodePieces(sentencepiece::SentencePieceProcessor const *self,std::vector< std::string > const &pieces){
+ return self->DecodePieces(pieces);
+ }
+SWIGINTERN sentencepiece::util::bytes sentencepiece_SentencePieceProcessor__DecodeIdsAsSerializedProto(sentencepiece::SentencePieceProcessor const *self,std::vector< int > const &ids){
+ CheckIds(ids, self->GetPieceSize());
+ return self->DecodeIdsAsSerializedProto(ids);
+ }
+SWIGINTERN sentencepiece::util::bytes sentencepiece_SentencePieceProcessor__DecodePiecesAsSerializedProto(sentencepiece::SentencePieceProcessor const *self,std::vector< std::string > const &pieces){
+ CheckIds(pieces, self->GetPieceSize());
+ return self->DecodePiecesAsSerializedProto(pieces);
+ }
+SWIGINTERN std::vector< std::string > sentencepiece_SentencePieceProcessor__DecodeIdsBatch(sentencepiece::SentencePieceProcessor const *self,std::vector< std::vector< int > > const &ins,int num_threads){
+ DEFINE_DECODE_BATCH_FUNC_IMPL(DecodeIds, int, std::string);
+ }
+SWIGINTERN BytesArray sentencepiece_SentencePieceProcessor__DecodeIdsAsSerializedProtoBatch(sentencepiece::SentencePieceProcessor const *self,std::vector< std::vector< int > > const &ins,int num_threads){
+ DEFINE_DECODE_BATCH_FUNC_IMPL(DecodeIdsAsSerializedProto, int,
+ sentencepiece::util::bytes);
+ }
+SWIGINTERN std::vector< std::string > sentencepiece_SentencePieceProcessor__DecodePiecesBatch(sentencepiece::SentencePieceProcessor const *self,std::vector< std::vector< std::string > > const &ins,int num_threads){
+ DEFINE_DECODE_BATCH_FUNC_IMPL(DecodePieces, std::string, std::string);
+ }
+SWIGINTERN BytesArray sentencepiece_SentencePieceProcessor__DecodePiecesAsSerializedProtoBatch(sentencepiece::SentencePieceProcessor const *self,std::vector< std::vector< std::string > > const &ins,int num_threads){
+ DEFINE_DECODE_BATCH_FUNC_IMPL(DecodePiecesAsSerializedProto, std::string,
+ sentencepiece::util::bytes);
+ }
+SWIGINTERN std::vector< std::vector< int > > sentencepiece_SentencePieceProcessor__NBestEncodeAsIds(sentencepiece::SentencePieceProcessor const *self,absl::string_view text,int nbest_size,bool add_bos,bool add_eos,bool reverse,bool emit_unk_piece){
auto idss = self->NBestEncodeAsIds(text, nbest_size);
for (auto &ids : idss) {
- RewriteIds(*self, &ids, add_bos, add_eos, reverse);
+ RewriteIds(*self, &ids, add_bos, add_eos, reverse, emit_unk_piece);
}
return idss;
}
-SWIGINTERN std::vector< std::vector< std::string > > sentencepiece_SentencePieceProcessor__NBestEncodeAsPieces(sentencepiece::SentencePieceProcessor *self,absl::string_view text,int nbest_size,bool add_bos,bool add_eos,bool reverse,bool emit_unk_piece){
+SWIGINTERN std::vector< std::vector< std::string > > sentencepiece_SentencePieceProcessor__NBestEncodeAsPieces(sentencepiece::SentencePieceProcessor const *self,absl::string_view text,int nbest_size,bool add_bos,bool add_eos,bool reverse,bool emit_unk_piece){
auto piecess = self->NBestEncodeAsPieces(text, nbest_size);
for (auto &pieces : piecess) {
- RewritePieces(*self, &pieces, add_bos, add_eos, reverse, emit_unk_piece);
+ RewriteIds(*self, &pieces, add_bos, add_eos, reverse, emit_unk_piece);
}
return piecess;
}
-SWIGINTERN std::vector< std::pair< std::vector< int >,float > > sentencepiece_SentencePieceProcessor__SampleEncodeAndScoreAsIds(sentencepiece::SentencePieceProcessor *self,absl::string_view text,int num_samples,float theta,bool wor,bool include_best,bool add_bos,bool add_eos,bool reverse){
+SWIGINTERN sentencepiece::util::bytes sentencepiece_SentencePieceProcessor__NBestEncodeAsSerializedProto(sentencepiece::SentencePieceProcessor const *self,absl::string_view text,int nbest_size,bool add_bos,bool add_eos,bool reverse,bool emit_unk_piece){
+ RewriteIds(*self, static_cast<sentencepiece::util::bytes *>(nullptr),
+ add_bos, add_eos, reverse, emit_unk_piece);
+ return self->NBestEncodeAsSerializedProto(text, nbest_size);
+ }
+SWIGINTERN std::vector< std::pair< std::vector< int >,float > > sentencepiece_SentencePieceProcessor__SampleEncodeAndScoreAsIds(sentencepiece::SentencePieceProcessor const *self,absl::string_view text,int num_samples,float theta,bool wor,bool include_best,bool add_bos,bool add_eos,bool reverse,bool emit_unk_piece){
auto idss = self->SampleEncodeAndScoreAsIds(text, num_samples,
theta, wor, include_best);
for (auto &ids : idss) {
- RewriteIds(*self, &ids.first, add_bos, add_eos, reverse);
+ RewriteIds(*self, &ids.first, add_bos, add_eos, reverse, emit_unk_piece);
}
return idss;
}
-SWIGINTERN std::vector< std::pair< std::vector< std::string >,float > > sentencepiece_SentencePieceProcessor__SampleEncodeAndScoreAsPieces(sentencepiece::SentencePieceProcessor *self,absl::string_view text,int num_samples,float theta,bool wor,bool include_best,bool add_bos,bool add_eos,bool reverse,bool emit_unk_piece){
+SWIGINTERN std::vector< std::pair< std::vector< std::string >,float > > sentencepiece_SentencePieceProcessor__SampleEncodeAndScoreAsPieces(sentencepiece::SentencePieceProcessor const *self,absl::string_view text,int num_samples,float theta,bool wor,bool include_best,bool add_bos,bool add_eos,bool reverse,bool emit_unk_piece){
auto piecess = self->SampleEncodeAndScoreAsPieces(text, num_samples,
theta, wor, include_best);
for (auto &pieces : piecess) {
- RewritePieces(*self, &pieces.first, add_bos, add_eos, reverse, emit_unk_piece);
+ RewriteIds(*self, &pieces.first, add_bos, add_eos, reverse, emit_unk_piece);
}
return piecess;
}
+SWIGINTERN float sentencepiece_SentencePieceProcessor__CalculateEntropy(sentencepiece::SentencePieceProcessor *self,absl::string_view text,float theta){
+ return self->CalculateEntropy(text, theta);
+ }
+SWIGINTERN std::vector< float > sentencepiece_SentencePieceProcessor__CalculateEntropyBatch(sentencepiece::SentencePieceProcessor *self,std::vector< absl::string_view > const &ins,float theta,int num_threads){
+ std::vector<float> outs(ins.size());
+ InitNumThreads(ins, &num_threads);
+ {
+ ThreadPool pool(ins.size());
+ for (int n = 0; n < num_threads; ++n) {
+ pool.Schedule([&, n]() {
+ for (size_t i = n; i < ins.size(); i += num_threads) {
+ outs[i] = self->CalculateEntropy(ins[i], theta);
+ }
+ });
+ }
+ }
+ return outs;
+ }
SWIGINTERN int
SWIG_AsVal_unsigned_SS_long (PyObject *obj, unsigned long *val)
for (size_t i = 0; i < size; ++i) {
const PyInputString ustring(PyList_GetItem(swig_obj[1], i));
if (ustring.IsAvalable()) {
- (*out)[i] = std::string(ustring.data(), ustring.size());
+ (*out)[i].assign(ustring.data(), ustring.size());
} else {
PyErr_SetString(PyExc_TypeError, "list must contain strings");
SWIG_fail;
}
-SWIGINTERN PyObject *_wrap_SentencePieceProcessor_EncodeAsPieces(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+SWIGINTERN PyObject *_wrap_SentencePieceProcessor_SampleEncodeAndScoreAsPieces(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
PyObject *resultobj = 0;
sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ;
absl::string_view arg2 ;
+ int arg3 ;
+ float arg4 ;
+ bool arg5 ;
+ bool arg6 ;
void *argp1 = 0 ;
int res1 = 0 ;
- PyObject *swig_obj[2] ;
- std::vector< std::string > result;
+ int val3 ;
+ int ecode3 = 0 ;
+ float val4 ;
+ int ecode4 = 0 ;
+ bool val5 ;
+ int ecode5 = 0 ;
+ bool val6 ;
+ int ecode6 = 0 ;
+ PyObject *swig_obj[6] ;
+ std::vector< std::pair< std::vector< std::string >,float > > result;
- if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor_EncodeAsPieces", 2, 2, swig_obj)) SWIG_fail;
+ if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor_SampleEncodeAndScoreAsPieces", 6, 6, swig_obj)) SWIG_fail;
res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 );
if (!SWIG_IsOK(res1)) {
- SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor_EncodeAsPieces" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'");
+ SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor_SampleEncodeAndScoreAsPieces" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'");
}
arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1);
{
resultobj = ustring.input_type();
arg2 = absl::string_view(ustring.data(), ustring.size());
}
+ ecode3 = SWIG_AsVal_int(swig_obj[2], &val3);
+ if (!SWIG_IsOK(ecode3)) {
+ SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "SentencePieceProcessor_SampleEncodeAndScoreAsPieces" "', argument " "3"" of type '" "int""'");
+ }
+ arg3 = static_cast< int >(val3);
+ ecode4 = SWIG_AsVal_float(swig_obj[3], &val4);
+ if (!SWIG_IsOK(ecode4)) {
+ SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "SentencePieceProcessor_SampleEncodeAndScoreAsPieces" "', argument " "4"" of type '" "float""'");
+ }
+ arg4 = static_cast< float >(val4);
+ ecode5 = SWIG_AsVal_bool(swig_obj[4], &val5);
+ if (!SWIG_IsOK(ecode5)) {
+ SWIG_exception_fail(SWIG_ArgError(ecode5), "in method '" "SentencePieceProcessor_SampleEncodeAndScoreAsPieces" "', argument " "5"" of type '" "bool""'");
+ }
+ arg5 = static_cast< bool >(val5);
+ ecode6 = SWIG_AsVal_bool(swig_obj[5], &val6);
+ if (!SWIG_IsOK(ecode6)) {
+ SWIG_exception_fail(SWIG_ArgError(ecode6), "in method '" "SentencePieceProcessor_SampleEncodeAndScoreAsPieces" "', argument " "6"" of type '" "bool""'");
+ }
+ arg6 = static_cast< bool >(val6);
{
try {
- result = ((sentencepiece::SentencePieceProcessor const *)arg1)->EncodeAsPieces(arg2);
+ result = ((sentencepiece::SentencePieceProcessor const *)arg1)->SampleEncodeAndScoreAsPieces(arg2,arg3,arg4,arg5,arg6);
ReleaseResultObject(resultobj);
}
catch (const sentencepiece::util::Status &status) {
PyObject *input_type = resultobj;
resultobj = PyList_New((&result)->size());
for (size_t i = 0; i < (&result)->size(); ++i) {
- PyList_SetItem(resultobj, i, MakePyOutputString(result[i], input_type));
+ PyObject *obj = PyList_New(result[i].first.size());
+ for (size_t j = 0; j < result[i].first.size(); ++j) {
+ PyList_SetItem(obj, j, MakePyOutputString(result[i].first[j], input_type));
+ }
+ PyList_SetItem(resultobj, i, PyTuple_Pack(2, obj, PyFloat_FromDouble(static_cast<double>(result[i].second))));
}
}
return resultobj;
}
-SWIGINTERN PyObject *_wrap_SentencePieceProcessor_EncodeAsIds(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+SWIGINTERN PyObject *_wrap_SentencePieceProcessor_SampleEncodeAndScoreAsIds(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
PyObject *resultobj = 0;
sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ;
absl::string_view arg2 ;
+ int arg3 ;
+ float arg4 ;
+ bool arg5 ;
+ bool arg6 ;
void *argp1 = 0 ;
int res1 = 0 ;
- PyObject *swig_obj[2] ;
- std::vector< int > result;
+ int val3 ;
+ int ecode3 = 0 ;
+ float val4 ;
+ int ecode4 = 0 ;
+ bool val5 ;
+ int ecode5 = 0 ;
+ bool val6 ;
+ int ecode6 = 0 ;
+ PyObject *swig_obj[6] ;
+ std::vector< std::pair< std::vector< int >,float > > result;
- if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor_EncodeAsIds", 2, 2, swig_obj)) SWIG_fail;
+ if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor_SampleEncodeAndScoreAsIds", 6, 6, swig_obj)) SWIG_fail;
res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 );
if (!SWIG_IsOK(res1)) {
- SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor_EncodeAsIds" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'");
+ SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor_SampleEncodeAndScoreAsIds" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'");
}
arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1);
{
resultobj = ustring.input_type();
arg2 = absl::string_view(ustring.data(), ustring.size());
}
+ ecode3 = SWIG_AsVal_int(swig_obj[2], &val3);
+ if (!SWIG_IsOK(ecode3)) {
+ SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "SentencePieceProcessor_SampleEncodeAndScoreAsIds" "', argument " "3"" of type '" "int""'");
+ }
+ arg3 = static_cast< int >(val3);
+ ecode4 = SWIG_AsVal_float(swig_obj[3], &val4);
+ if (!SWIG_IsOK(ecode4)) {
+ SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "SentencePieceProcessor_SampleEncodeAndScoreAsIds" "', argument " "4"" of type '" "float""'");
+ }
+ arg4 = static_cast< float >(val4);
+ ecode5 = SWIG_AsVal_bool(swig_obj[4], &val5);
+ if (!SWIG_IsOK(ecode5)) {
+ SWIG_exception_fail(SWIG_ArgError(ecode5), "in method '" "SentencePieceProcessor_SampleEncodeAndScoreAsIds" "', argument " "5"" of type '" "bool""'");
+ }
+ arg5 = static_cast< bool >(val5);
+ ecode6 = SWIG_AsVal_bool(swig_obj[5], &val6);
+ if (!SWIG_IsOK(ecode6)) {
+ SWIG_exception_fail(SWIG_ArgError(ecode6), "in method '" "SentencePieceProcessor_SampleEncodeAndScoreAsIds" "', argument " "6"" of type '" "bool""'");
+ }
+ arg6 = static_cast< bool >(val6);
{
try {
- result = ((sentencepiece::SentencePieceProcessor const *)arg1)->EncodeAsIds(arg2);
+ result = ((sentencepiece::SentencePieceProcessor const *)arg1)->SampleEncodeAndScoreAsIds(arg2,arg3,arg4,arg5,arg6);
ReleaseResultObject(resultobj);
}
catch (const sentencepiece::util::Status &status) {
{
resultobj = PyList_New((&result)->size());
for (size_t i = 0; i < (&result)->size(); ++i) {
- PyList_SetItem(resultobj, i, PyInt_FromLong(static_cast<long>(result[i])));
+ PyObject *obj = PyList_New(result[i].first.size());
+ for (size_t j = 0; j < result[i].first.size(); ++j) {
+ PyList_SetItem(obj, j, PyInt_FromLong(static_cast<long>(result[i].first[j])));
+ }
+ PyList_SetItem(resultobj, i, PyTuple_Pack(2, obj, PyFloat_FromDouble(static_cast<double>(result[i].second))));
}
}
return resultobj;
}
-SWIGINTERN PyObject *_wrap_SentencePieceProcessor_NBestEncodeAsPieces(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+SWIGINTERN PyObject *_wrap_SentencePieceProcessor_CalculateEntropy(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
PyObject *resultobj = 0;
sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ;
absl::string_view arg2 ;
- int arg3 ;
+ float arg3 ;
void *argp1 = 0 ;
int res1 = 0 ;
- int val3 ;
+ float val3 ;
int ecode3 = 0 ;
PyObject *swig_obj[3] ;
- std::vector< std::vector< std::string > > result;
+ float result;
- if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor_NBestEncodeAsPieces", 3, 3, swig_obj)) SWIG_fail;
+ if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor_CalculateEntropy", 3, 3, swig_obj)) SWIG_fail;
res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 );
if (!SWIG_IsOK(res1)) {
- SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor_NBestEncodeAsPieces" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'");
+ SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor_CalculateEntropy" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'");
}
arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1);
{
resultobj = ustring.input_type();
arg2 = absl::string_view(ustring.data(), ustring.size());
}
- ecode3 = SWIG_AsVal_int(swig_obj[2], &val3);
+ ecode3 = SWIG_AsVal_float(swig_obj[2], &val3);
if (!SWIG_IsOK(ecode3)) {
- SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "SentencePieceProcessor_NBestEncodeAsPieces" "', argument " "3"" of type '" "int""'");
+ SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "SentencePieceProcessor_CalculateEntropy" "', argument " "3"" of type '" "float""'");
}
- arg3 = static_cast< int >(val3);
+ arg3 = static_cast< float >(val3);
{
try {
- result = ((sentencepiece::SentencePieceProcessor const *)arg1)->NBestEncodeAsPieces(arg2,arg3);
+ result = (float)((sentencepiece::SentencePieceProcessor const *)arg1)->CalculateEntropy(arg2,arg3);
ReleaseResultObject(resultobj);
}
catch (const sentencepiece::util::Status &status) {
SWIG_exception(ToSwigError(status.code()), status.ToString().c_str());
}
}
- {
- PyObject *input_type = resultobj;
- resultobj = PyList_New((&result)->size());
- for (size_t i = 0; i < (&result)->size(); ++i) {
- PyObject *obj = PyList_New(result[i].size());
- for (size_t j = 0; j < result[i].size(); ++j) {
- PyList_SetItem(obj, j, MakePyOutputString(result[i][j], input_type));
- }
- PyList_SetItem(resultobj, i, obj);
- }
- }
+ resultobj = SWIG_From_float(static_cast< float >(result));
return resultobj;
fail:
return NULL;
}
-SWIGINTERN PyObject *_wrap_SentencePieceProcessor_NBestEncodeAsIds(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+SWIGINTERN PyObject *_wrap_SentencePieceProcessor_GetPieceSize(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
PyObject *resultobj = 0;
sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ;
- absl::string_view arg2 ;
- int arg3 ;
void *argp1 = 0 ;
int res1 = 0 ;
- int val3 ;
- int ecode3 = 0 ;
- PyObject *swig_obj[3] ;
- std::vector< std::vector< int > > result;
+ PyObject *swig_obj[1] ;
+ int result;
- if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor_NBestEncodeAsIds", 3, 3, swig_obj)) SWIG_fail;
+ if (!args) SWIG_fail;
+ swig_obj[0] = args;
res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 );
if (!SWIG_IsOK(res1)) {
- SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor_NBestEncodeAsIds" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'");
+ SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor_GetPieceSize" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'");
}
arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1);
- {
- const PyInputString ustring(swig_obj[1]);
- if (!ustring.IsAvalable()) {
- PyErr_SetString(PyExc_TypeError, "not a string");
- SWIG_fail;
- }
- resultobj = ustring.input_type();
- arg2 = absl::string_view(ustring.data(), ustring.size());
- }
- ecode3 = SWIG_AsVal_int(swig_obj[2], &val3);
- if (!SWIG_IsOK(ecode3)) {
- SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "SentencePieceProcessor_NBestEncodeAsIds" "', argument " "3"" of type '" "int""'");
- }
- arg3 = static_cast< int >(val3);
{
try {
- result = ((sentencepiece::SentencePieceProcessor const *)arg1)->NBestEncodeAsIds(arg2,arg3);
+ result = (int)((sentencepiece::SentencePieceProcessor const *)arg1)->GetPieceSize();
ReleaseResultObject(resultobj);
}
catch (const sentencepiece::util::Status &status) {
SWIG_exception(ToSwigError(status.code()), status.ToString().c_str());
}
}
- {
- resultobj = PyList_New((&result)->size());
- for (size_t i = 0; i < (&result)->size(); ++i) {
- PyObject *obj = PyList_New(result[i].size());
- for (size_t j = 0; j < result[i].size(); ++j) {
- PyList_SetItem(obj, j, PyInt_FromLong(static_cast<long>(result[i][j])));
- }
- PyList_SetItem(resultobj, i, obj);
- }
- }
+ resultobj = SWIG_From_int(static_cast< int >(result));
return resultobj;
fail:
return NULL;
}
-SWIGINTERN PyObject *_wrap_SentencePieceProcessor_SampleEncodeAsPieces(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+SWIGINTERN PyObject *_wrap_SentencePieceProcessor_PieceToId(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
PyObject *resultobj = 0;
sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ;
absl::string_view arg2 ;
- int arg3 ;
- float arg4 ;
void *argp1 = 0 ;
int res1 = 0 ;
- int val3 ;
- int ecode3 = 0 ;
- float val4 ;
- int ecode4 = 0 ;
- PyObject *swig_obj[4] ;
- std::vector< std::string > result;
+ PyObject *swig_obj[2] ;
+ int result;
- if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor_SampleEncodeAsPieces", 4, 4, swig_obj)) SWIG_fail;
+ if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor_PieceToId", 2, 2, swig_obj)) SWIG_fail;
res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 );
if (!SWIG_IsOK(res1)) {
- SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor_SampleEncodeAsPieces" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'");
+ SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor_PieceToId" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'");
}
arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1);
{
resultobj = ustring.input_type();
arg2 = absl::string_view(ustring.data(), ustring.size());
}
- ecode3 = SWIG_AsVal_int(swig_obj[2], &val3);
- if (!SWIG_IsOK(ecode3)) {
- SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "SentencePieceProcessor_SampleEncodeAsPieces" "', argument " "3"" of type '" "int""'");
- }
- arg3 = static_cast< int >(val3);
- ecode4 = SWIG_AsVal_float(swig_obj[3], &val4);
- if (!SWIG_IsOK(ecode4)) {
- SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "SentencePieceProcessor_SampleEncodeAsPieces" "', argument " "4"" of type '" "float""'");
- }
- arg4 = static_cast< float >(val4);
{
try {
- result = ((sentencepiece::SentencePieceProcessor const *)arg1)->SampleEncodeAsPieces(arg2,arg3,arg4);
+ result = (int)((sentencepiece::SentencePieceProcessor const *)arg1)->PieceToId(arg2);
ReleaseResultObject(resultobj);
}
catch (const sentencepiece::util::Status &status) {
SWIG_exception(ToSwigError(status.code()), status.ToString().c_str());
}
}
- {
- PyObject *input_type = resultobj;
- resultobj = PyList_New((&result)->size());
- for (size_t i = 0; i < (&result)->size(); ++i) {
- PyList_SetItem(resultobj, i, MakePyOutputString(result[i], input_type));
- }
- }
+ resultobj = SWIG_From_int(static_cast< int >(result));
return resultobj;
fail:
return NULL;
}
-SWIGINTERN PyObject *_wrap_SentencePieceProcessor_SampleEncodeAsIds(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+SWIGINTERN PyObject *_wrap_SentencePieceProcessor_IdToPiece(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
PyObject *resultobj = 0;
sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ;
- absl::string_view arg2 ;
- int arg3 ;
- float arg4 ;
+ int arg2 ;
void *argp1 = 0 ;
int res1 = 0 ;
- int val3 ;
- int ecode3 = 0 ;
- float val4 ;
- int ecode4 = 0 ;
- PyObject *swig_obj[4] ;
- std::vector< int > result;
+ int val2 ;
+ int ecode2 = 0 ;
+ PyObject *swig_obj[2] ;
+ std::string *result = 0 ;
- if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor_SampleEncodeAsIds", 4, 4, swig_obj)) SWIG_fail;
+ if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor_IdToPiece", 2, 2, swig_obj)) SWIG_fail;
res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 );
if (!SWIG_IsOK(res1)) {
- SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor_SampleEncodeAsIds" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'");
+ SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor_IdToPiece" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'");
}
arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1);
- {
- const PyInputString ustring(swig_obj[1]);
- if (!ustring.IsAvalable()) {
- PyErr_SetString(PyExc_TypeError, "not a string");
- SWIG_fail;
- }
- resultobj = ustring.input_type();
- arg2 = absl::string_view(ustring.data(), ustring.size());
- }
- ecode3 = SWIG_AsVal_int(swig_obj[2], &val3);
- if (!SWIG_IsOK(ecode3)) {
- SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "SentencePieceProcessor_SampleEncodeAsIds" "', argument " "3"" of type '" "int""'");
- }
- arg3 = static_cast< int >(val3);
- ecode4 = SWIG_AsVal_float(swig_obj[3], &val4);
- if (!SWIG_IsOK(ecode4)) {
- SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "SentencePieceProcessor_SampleEncodeAsIds" "', argument " "4"" of type '" "float""'");
+ ecode2 = SWIG_AsVal_int(swig_obj[1], &val2);
+ if (!SWIG_IsOK(ecode2)) {
+ SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "SentencePieceProcessor_IdToPiece" "', argument " "2"" of type '" "int""'");
}
- arg4 = static_cast< float >(val4);
+ arg2 = static_cast< int >(val2);
{
try {
- result = ((sentencepiece::SentencePieceProcessor const *)arg1)->SampleEncodeAsIds(arg2,arg3,arg4);
+ result = (std::string *) &((sentencepiece::SentencePieceProcessor const *)arg1)->IdToPiece(arg2);
ReleaseResultObject(resultobj);
}
catch (const sentencepiece::util::Status &status) {
}
}
{
- resultobj = PyList_New((&result)->size());
- for (size_t i = 0; i < (&result)->size(); ++i) {
- PyList_SetItem(resultobj, i, PyInt_FromLong(static_cast<long>(result[i])));
- }
+ PyObject *input_type = resultobj;
+ resultobj = MakePyOutputString(*result, input_type);
}
return resultobj;
fail:
}
-SWIGINTERN PyObject *_wrap_SentencePieceProcessor_SampleEncodeAndScoreAsPieces(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+SWIGINTERN PyObject *_wrap_SentencePieceProcessor_GetScore(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
PyObject *resultobj = 0;
sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ;
- absl::string_view arg2 ;
- int arg3 ;
- float arg4 ;
- bool arg5 ;
- bool arg6 ;
+ int arg2 ;
void *argp1 = 0 ;
int res1 = 0 ;
- int val3 ;
- int ecode3 = 0 ;
- float val4 ;
- int ecode4 = 0 ;
- bool val5 ;
- int ecode5 = 0 ;
- bool val6 ;
- int ecode6 = 0 ;
- PyObject *swig_obj[6] ;
- std::vector< std::pair< std::vector< std::string >,float > > result;
+ int val2 ;
+ int ecode2 = 0 ;
+ PyObject *swig_obj[2] ;
+ float result;
- if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor_SampleEncodeAndScoreAsPieces", 6, 6, swig_obj)) SWIG_fail;
+ if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor_GetScore", 2, 2, swig_obj)) SWIG_fail;
res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 );
if (!SWIG_IsOK(res1)) {
- SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor_SampleEncodeAndScoreAsPieces" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'");
+ SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor_GetScore" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'");
}
arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1);
- {
- const PyInputString ustring(swig_obj[1]);
- if (!ustring.IsAvalable()) {
- PyErr_SetString(PyExc_TypeError, "not a string");
- SWIG_fail;
- }
- resultobj = ustring.input_type();
- arg2 = absl::string_view(ustring.data(), ustring.size());
- }
- ecode3 = SWIG_AsVal_int(swig_obj[2], &val3);
- if (!SWIG_IsOK(ecode3)) {
- SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "SentencePieceProcessor_SampleEncodeAndScoreAsPieces" "', argument " "3"" of type '" "int""'");
- }
- arg3 = static_cast< int >(val3);
- ecode4 = SWIG_AsVal_float(swig_obj[3], &val4);
- if (!SWIG_IsOK(ecode4)) {
- SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "SentencePieceProcessor_SampleEncodeAndScoreAsPieces" "', argument " "4"" of type '" "float""'");
- }
- arg4 = static_cast< float >(val4);
- ecode5 = SWIG_AsVal_bool(swig_obj[4], &val5);
- if (!SWIG_IsOK(ecode5)) {
- SWIG_exception_fail(SWIG_ArgError(ecode5), "in method '" "SentencePieceProcessor_SampleEncodeAndScoreAsPieces" "', argument " "5"" of type '" "bool""'");
- }
- arg5 = static_cast< bool >(val5);
- ecode6 = SWIG_AsVal_bool(swig_obj[5], &val6);
- if (!SWIG_IsOK(ecode6)) {
- SWIG_exception_fail(SWIG_ArgError(ecode6), "in method '" "SentencePieceProcessor_SampleEncodeAndScoreAsPieces" "', argument " "6"" of type '" "bool""'");
+ ecode2 = SWIG_AsVal_int(swig_obj[1], &val2);
+ if (!SWIG_IsOK(ecode2)) {
+ SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "SentencePieceProcessor_GetScore" "', argument " "2"" of type '" "int""'");
}
- arg6 = static_cast< bool >(val6);
+ arg2 = static_cast< int >(val2);
{
try {
- result = ((sentencepiece::SentencePieceProcessor const *)arg1)->SampleEncodeAndScoreAsPieces(arg2,arg3,arg4,arg5,arg6);
+ result = (float)((sentencepiece::SentencePieceProcessor const *)arg1)->GetScore(arg2);
ReleaseResultObject(resultobj);
}
catch (const sentencepiece::util::Status &status) {
SWIG_exception(ToSwigError(status.code()), status.ToString().c_str());
}
}
- {
- PyObject *input_type = resultobj;
- resultobj = PyList_New((&result)->size());
- for (size_t i = 0; i < (&result)->size(); ++i) {
- PyObject *obj = PyList_New(result[i].first.size());
- for (size_t j = 0; j < result[i].first.size(); ++j) {
- PyList_SetItem(obj, j, MakePyOutputString(result[i].first[j], input_type));
- }
- PyList_SetItem(resultobj, i, PyTuple_Pack(2, obj, PyFloat_FromDouble(static_cast<double>(result[i].second))));
- }
- }
+ resultobj = SWIG_From_float(static_cast< float >(result));
return resultobj;
fail:
return NULL;
}
-SWIGINTERN PyObject *_wrap_SentencePieceProcessor_SampleEncodeAndScoreAsIds(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+SWIGINTERN PyObject *_wrap_SentencePieceProcessor_IsUnknown(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
PyObject *resultobj = 0;
sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ;
- absl::string_view arg2 ;
- int arg3 ;
- float arg4 ;
- bool arg5 ;
- bool arg6 ;
+ int arg2 ;
void *argp1 = 0 ;
int res1 = 0 ;
- int val3 ;
- int ecode3 = 0 ;
- float val4 ;
- int ecode4 = 0 ;
- bool val5 ;
- int ecode5 = 0 ;
- bool val6 ;
- int ecode6 = 0 ;
- PyObject *swig_obj[6] ;
- std::vector< std::pair< std::vector< int >,float > > result;
+ int val2 ;
+ int ecode2 = 0 ;
+ PyObject *swig_obj[2] ;
+ bool result;
- if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor_SampleEncodeAndScoreAsIds", 6, 6, swig_obj)) SWIG_fail;
+ if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor_IsUnknown", 2, 2, swig_obj)) SWIG_fail;
res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 );
if (!SWIG_IsOK(res1)) {
- SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor_SampleEncodeAndScoreAsIds" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'");
+ SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor_IsUnknown" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'");
}
arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1);
- {
- const PyInputString ustring(swig_obj[1]);
- if (!ustring.IsAvalable()) {
- PyErr_SetString(PyExc_TypeError, "not a string");
- SWIG_fail;
- }
- resultobj = ustring.input_type();
- arg2 = absl::string_view(ustring.data(), ustring.size());
- }
- ecode3 = SWIG_AsVal_int(swig_obj[2], &val3);
- if (!SWIG_IsOK(ecode3)) {
- SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "SentencePieceProcessor_SampleEncodeAndScoreAsIds" "', argument " "3"" of type '" "int""'");
- }
- arg3 = static_cast< int >(val3);
- ecode4 = SWIG_AsVal_float(swig_obj[3], &val4);
- if (!SWIG_IsOK(ecode4)) {
- SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "SentencePieceProcessor_SampleEncodeAndScoreAsIds" "', argument " "4"" of type '" "float""'");
- }
- arg4 = static_cast< float >(val4);
- ecode5 = SWIG_AsVal_bool(swig_obj[4], &val5);
- if (!SWIG_IsOK(ecode5)) {
- SWIG_exception_fail(SWIG_ArgError(ecode5), "in method '" "SentencePieceProcessor_SampleEncodeAndScoreAsIds" "', argument " "5"" of type '" "bool""'");
- }
- arg5 = static_cast< bool >(val5);
- ecode6 = SWIG_AsVal_bool(swig_obj[5], &val6);
- if (!SWIG_IsOK(ecode6)) {
- SWIG_exception_fail(SWIG_ArgError(ecode6), "in method '" "SentencePieceProcessor_SampleEncodeAndScoreAsIds" "', argument " "6"" of type '" "bool""'");
+ ecode2 = SWIG_AsVal_int(swig_obj[1], &val2);
+ if (!SWIG_IsOK(ecode2)) {
+ SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "SentencePieceProcessor_IsUnknown" "', argument " "2"" of type '" "int""'");
}
- arg6 = static_cast< bool >(val6);
+ arg2 = static_cast< int >(val2);
{
try {
- result = ((sentencepiece::SentencePieceProcessor const *)arg1)->SampleEncodeAndScoreAsIds(arg2,arg3,arg4,arg5,arg6);
+ result = (bool)((sentencepiece::SentencePieceProcessor const *)arg1)->IsUnknown(arg2);
ReleaseResultObject(resultobj);
}
catch (const sentencepiece::util::Status &status) {
SWIG_exception(ToSwigError(status.code()), status.ToString().c_str());
}
}
- {
- resultobj = PyList_New((&result)->size());
- for (size_t i = 0; i < (&result)->size(); ++i) {
- PyObject *obj = PyList_New(result[i].first.size());
- for (size_t j = 0; j < result[i].first.size(); ++j) {
- PyList_SetItem(obj, j, PyInt_FromLong(static_cast<long>(result[i].first[j])));
- }
- PyList_SetItem(resultobj, i, PyTuple_Pack(2, obj, PyFloat_FromDouble(static_cast<double>(result[i].second))));
- }
- }
+ resultobj = SWIG_From_bool(static_cast< bool >(result));
return resultobj;
fail:
return NULL;
}
-SWIGINTERN PyObject *_wrap_SentencePieceProcessor_DecodePieces(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+SWIGINTERN PyObject *_wrap_SentencePieceProcessor_IsControl(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
PyObject *resultobj = 0;
sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ;
- std::vector< std::string > *arg2 = 0 ;
+ int arg2 ;
void *argp1 = 0 ;
int res1 = 0 ;
+ int val2 ;
+ int ecode2 = 0 ;
PyObject *swig_obj[2] ;
- std::string result;
+ bool result;
- if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor_DecodePieces", 2, 2, swig_obj)) SWIG_fail;
+ if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor_IsControl", 2, 2, swig_obj)) SWIG_fail;
res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 );
if (!SWIG_IsOK(res1)) {
- SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor_DecodePieces" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'");
+ SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor_IsControl" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'");
}
arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1);
- {
- std::vector<std::string> *out = nullptr;
- if (PyList_Check(swig_obj[1])) {
- const size_t size = PyList_Size(swig_obj[1]);
- out = new std::vector<std::string>(size);
- for (size_t i = 0; i < size; ++i) {
- const PyInputString ustring(PyList_GetItem(swig_obj[1], i));
- if (ustring.IsAvalable()) {
- (*out)[i] = std::string(ustring.data(), ustring.size());
- } else {
- PyErr_SetString(PyExc_TypeError, "list must contain strings");
- SWIG_fail;
- }
- resultobj = ustring.input_type();
- }
- } else {
- PyErr_SetString(PyExc_TypeError, "not a list");
- SWIG_fail;
- }
- arg2 = out;
- }
+ ecode2 = SWIG_AsVal_int(swig_obj[1], &val2);
+ if (!SWIG_IsOK(ecode2)) {
+ SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "SentencePieceProcessor_IsControl" "', argument " "2"" of type '" "int""'");
+ }
+ arg2 = static_cast< int >(val2);
{
try {
- result = ((sentencepiece::SentencePieceProcessor const *)arg1)->DecodePieces((std::vector< std::string > const &)*arg2);
+ result = (bool)((sentencepiece::SentencePieceProcessor const *)arg1)->IsControl(arg2);
ReleaseResultObject(resultobj);
}
catch (const sentencepiece::util::Status &status) {
SWIG_exception(ToSwigError(status.code()), status.ToString().c_str());
}
}
- {
- PyObject *input_type = resultobj;
- resultobj = MakePyOutputString(result, input_type);
- }
- {
- delete arg2;
- }
+ resultobj = SWIG_From_bool(static_cast< bool >(result));
return resultobj;
fail:
- {
- delete arg2;
- }
return NULL;
}
-SWIGINTERN PyObject *_wrap_SentencePieceProcessor_CalculateEntropy(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+SWIGINTERN PyObject *_wrap_SentencePieceProcessor_IsUnused(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
PyObject *resultobj = 0;
sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ;
- absl::string_view arg2 ;
- float arg3 ;
+ int arg2 ;
void *argp1 = 0 ;
int res1 = 0 ;
- float val3 ;
- int ecode3 = 0 ;
- PyObject *swig_obj[3] ;
- float result;
+ int val2 ;
+ int ecode2 = 0 ;
+ PyObject *swig_obj[2] ;
+ bool result;
- if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor_CalculateEntropy", 3, 3, swig_obj)) SWIG_fail;
+ if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor_IsUnused", 2, 2, swig_obj)) SWIG_fail;
res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 );
if (!SWIG_IsOK(res1)) {
- SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor_CalculateEntropy" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'");
+ SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor_IsUnused" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'");
}
arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1);
- {
- const PyInputString ustring(swig_obj[1]);
- if (!ustring.IsAvalable()) {
- PyErr_SetString(PyExc_TypeError, "not a string");
- SWIG_fail;
- }
- resultobj = ustring.input_type();
- arg2 = absl::string_view(ustring.data(), ustring.size());
- }
- ecode3 = SWIG_AsVal_float(swig_obj[2], &val3);
- if (!SWIG_IsOK(ecode3)) {
- SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "SentencePieceProcessor_CalculateEntropy" "', argument " "3"" of type '" "float""'");
+ ecode2 = SWIG_AsVal_int(swig_obj[1], &val2);
+ if (!SWIG_IsOK(ecode2)) {
+ SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "SentencePieceProcessor_IsUnused" "', argument " "2"" of type '" "int""'");
}
- arg3 = static_cast< float >(val3);
+ arg2 = static_cast< int >(val2);
{
try {
- result = (float)((sentencepiece::SentencePieceProcessor const *)arg1)->CalculateEntropy(arg2,arg3);
+ result = (bool)((sentencepiece::SentencePieceProcessor const *)arg1)->IsUnused(arg2);
ReleaseResultObject(resultobj);
}
catch (const sentencepiece::util::Status &status) {
SWIG_exception(ToSwigError(status.code()), status.ToString().c_str());
}
}
- resultobj = SWIG_From_float(static_cast< float >(result));
+ resultobj = SWIG_From_bool(static_cast< bool >(result));
return resultobj;
fail:
return NULL;
}
-SWIGINTERN PyObject *_wrap_SentencePieceProcessor_EncodeAsSerializedProto(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+SWIGINTERN PyObject *_wrap_SentencePieceProcessor_IsByte(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
PyObject *resultobj = 0;
sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ;
- absl::string_view arg2 ;
+ int arg2 ;
void *argp1 = 0 ;
int res1 = 0 ;
+ int val2 ;
+ int ecode2 = 0 ;
PyObject *swig_obj[2] ;
- sentencepiece::util::bytes result;
+ bool result;
- if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor_EncodeAsSerializedProto", 2, 2, swig_obj)) SWIG_fail;
+ if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor_IsByte", 2, 2, swig_obj)) SWIG_fail;
res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 );
if (!SWIG_IsOK(res1)) {
- SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor_EncodeAsSerializedProto" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'");
+ SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor_IsByte" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'");
}
arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1);
- {
- const PyInputString ustring(swig_obj[1]);
- if (!ustring.IsAvalable()) {
- PyErr_SetString(PyExc_TypeError, "not a string");
- SWIG_fail;
- }
- resultobj = ustring.input_type();
- arg2 = absl::string_view(ustring.data(), ustring.size());
- }
+ ecode2 = SWIG_AsVal_int(swig_obj[1], &val2);
+ if (!SWIG_IsOK(ecode2)) {
+ SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "SentencePieceProcessor_IsByte" "', argument " "2"" of type '" "int""'");
+ }
+ arg2 = static_cast< int >(val2);
{
try {
- result = ((sentencepiece::SentencePieceProcessor const *)arg1)->EncodeAsSerializedProto(arg2);
+ result = (bool)((sentencepiece::SentencePieceProcessor const *)arg1)->IsByte(arg2);
ReleaseResultObject(resultobj);
}
catch (const sentencepiece::util::Status &status) {
SWIG_exception(ToSwigError(status.code()), status.ToString().c_str());
}
}
- {
- resultobj = MakePyOutputBytes(result);
- }
+ resultobj = SWIG_From_bool(static_cast< bool >(result));
return resultobj;
fail:
return NULL;
}
-SWIGINTERN PyObject *_wrap_SentencePieceProcessor_SampleEncodeAsSerializedProto(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+SWIGINTERN PyObject *_wrap_SentencePieceProcessor_unk_id(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
PyObject *resultobj = 0;
sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ;
- absl::string_view arg2 ;
- int arg3 ;
- float arg4 ;
void *argp1 = 0 ;
int res1 = 0 ;
- int val3 ;
- int ecode3 = 0 ;
- float val4 ;
- int ecode4 = 0 ;
- PyObject *swig_obj[4] ;
- sentencepiece::util::bytes result;
+ PyObject *swig_obj[1] ;
+ int result;
- if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor_SampleEncodeAsSerializedProto", 4, 4, swig_obj)) SWIG_fail;
+ if (!args) SWIG_fail;
+ swig_obj[0] = args;
res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 );
if (!SWIG_IsOK(res1)) {
- SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor_SampleEncodeAsSerializedProto" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'");
+ SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor_unk_id" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'");
}
arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1);
- {
- const PyInputString ustring(swig_obj[1]);
- if (!ustring.IsAvalable()) {
- PyErr_SetString(PyExc_TypeError, "not a string");
- SWIG_fail;
- }
- resultobj = ustring.input_type();
- arg2 = absl::string_view(ustring.data(), ustring.size());
- }
- ecode3 = SWIG_AsVal_int(swig_obj[2], &val3);
- if (!SWIG_IsOK(ecode3)) {
- SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "SentencePieceProcessor_SampleEncodeAsSerializedProto" "', argument " "3"" of type '" "int""'");
- }
- arg3 = static_cast< int >(val3);
- ecode4 = SWIG_AsVal_float(swig_obj[3], &val4);
- if (!SWIG_IsOK(ecode4)) {
- SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "SentencePieceProcessor_SampleEncodeAsSerializedProto" "', argument " "4"" of type '" "float""'");
- }
- arg4 = static_cast< float >(val4);
{
try {
- result = ((sentencepiece::SentencePieceProcessor const *)arg1)->SampleEncodeAsSerializedProto(arg2,arg3,arg4);
+ result = (int)((sentencepiece::SentencePieceProcessor const *)arg1)->unk_id();
ReleaseResultObject(resultobj);
}
catch (const sentencepiece::util::Status &status) {
SWIG_exception(ToSwigError(status.code()), status.ToString().c_str());
}
}
- {
- resultobj = MakePyOutputBytes(result);
- }
+ resultobj = SWIG_From_int(static_cast< int >(result));
return resultobj;
fail:
return NULL;
}
-SWIGINTERN PyObject *_wrap_SentencePieceProcessor_NBestEncodeAsSerializedProto(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+SWIGINTERN PyObject *_wrap_SentencePieceProcessor_bos_id(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
PyObject *resultobj = 0;
sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ;
- absl::string_view arg2 ;
- int arg3 ;
void *argp1 = 0 ;
int res1 = 0 ;
- int val3 ;
- int ecode3 = 0 ;
- PyObject *swig_obj[3] ;
- sentencepiece::util::bytes result;
+ PyObject *swig_obj[1] ;
+ int result;
- if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor_NBestEncodeAsSerializedProto", 3, 3, swig_obj)) SWIG_fail;
+ if (!args) SWIG_fail;
+ swig_obj[0] = args;
res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 );
if (!SWIG_IsOK(res1)) {
- SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor_NBestEncodeAsSerializedProto" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'");
+ SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor_bos_id" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'");
}
arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1);
- {
- const PyInputString ustring(swig_obj[1]);
- if (!ustring.IsAvalable()) {
- PyErr_SetString(PyExc_TypeError, "not a string");
- SWIG_fail;
- }
- resultobj = ustring.input_type();
- arg2 = absl::string_view(ustring.data(), ustring.size());
- }
- ecode3 = SWIG_AsVal_int(swig_obj[2], &val3);
- if (!SWIG_IsOK(ecode3)) {
- SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "SentencePieceProcessor_NBestEncodeAsSerializedProto" "', argument " "3"" of type '" "int""'");
- }
- arg3 = static_cast< int >(val3);
{
try {
- result = ((sentencepiece::SentencePieceProcessor const *)arg1)->NBestEncodeAsSerializedProto(arg2,arg3);
+ result = (int)((sentencepiece::SentencePieceProcessor const *)arg1)->bos_id();
ReleaseResultObject(resultobj);
}
catch (const sentencepiece::util::Status &status) {
SWIG_exception(ToSwigError(status.code()), status.ToString().c_str());
}
}
- {
- resultobj = MakePyOutputBytes(result);
- }
+ resultobj = SWIG_From_int(static_cast< int >(result));
return resultobj;
fail:
return NULL;
}
-SWIGINTERN PyObject *_wrap_SentencePieceProcessor_DecodePiecesAsSerializedProto(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+SWIGINTERN PyObject *_wrap_SentencePieceProcessor_eos_id(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
PyObject *resultobj = 0;
sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ;
- std::vector< std::string > *arg2 = 0 ;
void *argp1 = 0 ;
int res1 = 0 ;
- PyObject *swig_obj[2] ;
- sentencepiece::util::bytes result;
+ PyObject *swig_obj[1] ;
+ int result;
- if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor_DecodePiecesAsSerializedProto", 2, 2, swig_obj)) SWIG_fail;
+ if (!args) SWIG_fail;
+ swig_obj[0] = args;
res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 );
if (!SWIG_IsOK(res1)) {
- SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor_DecodePiecesAsSerializedProto" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'");
+ SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor_eos_id" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'");
}
arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1);
- {
- std::vector<std::string> *out = nullptr;
- if (PyList_Check(swig_obj[1])) {
- const size_t size = PyList_Size(swig_obj[1]);
- out = new std::vector<std::string>(size);
- for (size_t i = 0; i < size; ++i) {
- const PyInputString ustring(PyList_GetItem(swig_obj[1], i));
- if (ustring.IsAvalable()) {
- (*out)[i] = std::string(ustring.data(), ustring.size());
- } else {
- PyErr_SetString(PyExc_TypeError, "list must contain strings");
- SWIG_fail;
- }
- resultobj = ustring.input_type();
- }
- } else {
- PyErr_SetString(PyExc_TypeError, "not a list");
- SWIG_fail;
- }
- arg2 = out;
- }
{
try {
- result = ((sentencepiece::SentencePieceProcessor const *)arg1)->DecodePiecesAsSerializedProto((std::vector< std::string > const &)*arg2);
+ result = (int)((sentencepiece::SentencePieceProcessor const *)arg1)->eos_id();
ReleaseResultObject(resultobj);
}
catch (const sentencepiece::util::Status &status) {
SWIG_exception(ToSwigError(status.code()), status.ToString().c_str());
}
}
- {
- resultobj = MakePyOutputBytes(result);
- }
- {
- delete arg2;
- }
+ resultobj = SWIG_From_int(static_cast< int >(result));
return resultobj;
fail:
- {
- delete arg2;
- }
return NULL;
}
-SWIGINTERN PyObject *_wrap_SentencePieceProcessor_GetPieceSize(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+SWIGINTERN PyObject *_wrap_SentencePieceProcessor_pad_id(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
PyObject *resultobj = 0;
sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ;
void *argp1 = 0 ;
swig_obj[0] = args;
res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 );
if (!SWIG_IsOK(res1)) {
- SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor_GetPieceSize" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'");
+ SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor_pad_id" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'");
}
arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1);
{
try {
- result = (int)((sentencepiece::SentencePieceProcessor const *)arg1)->GetPieceSize();
+ result = (int)((sentencepiece::SentencePieceProcessor const *)arg1)->pad_id();
ReleaseResultObject(resultobj);
}
catch (const sentencepiece::util::Status &status) {
}
-SWIGINTERN PyObject *_wrap_SentencePieceProcessor_PieceToId(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+SWIGINTERN PyObject *_wrap_SentencePieceProcessor_serialized_model_proto(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
PyObject *resultobj = 0;
sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ;
- absl::string_view arg2 ;
void *argp1 = 0 ;
int res1 = 0 ;
- PyObject *swig_obj[2] ;
- int result;
+ PyObject *swig_obj[1] ;
+ sentencepiece::util::bytes result;
- if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor_PieceToId", 2, 2, swig_obj)) SWIG_fail;
+ if (!args) SWIG_fail;
+ swig_obj[0] = args;
res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 );
if (!SWIG_IsOK(res1)) {
- SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor_PieceToId" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'");
+ SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor_serialized_model_proto" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'");
}
arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1);
- {
- const PyInputString ustring(swig_obj[1]);
- if (!ustring.IsAvalable()) {
- PyErr_SetString(PyExc_TypeError, "not a string");
- SWIG_fail;
- }
- resultobj = ustring.input_type();
- arg2 = absl::string_view(ustring.data(), ustring.size());
- }
{
try {
- result = (int)((sentencepiece::SentencePieceProcessor const *)arg1)->PieceToId(arg2);
+ result = ((sentencepiece::SentencePieceProcessor const *)arg1)->serialized_model_proto();
ReleaseResultObject(resultobj);
}
catch (const sentencepiece::util::Status &status) {
SWIG_exception(ToSwigError(status.code()), status.ToString().c_str());
}
}
- resultobj = SWIG_From_int(static_cast< int >(result));
+ {
+ resultobj = MakePyOutputBytes(result);
+ }
return resultobj;
fail:
return NULL;
}
-SWIGINTERN PyObject *_wrap_SentencePieceProcessor_IdToPiece(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+SWIGINTERN PyObject *_wrap_SentencePieceProcessor_LoadFromFile(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
PyObject *resultobj = 0;
sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ;
- int arg2 ;
+ absl::string_view arg2 ;
void *argp1 = 0 ;
int res1 = 0 ;
- int val2 ;
- int ecode2 = 0 ;
PyObject *swig_obj[2] ;
- std::string *result = 0 ;
+ sentencepiece::util::Status result;
- if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor_IdToPiece", 2, 2, swig_obj)) SWIG_fail;
+ if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor_LoadFromFile", 2, 2, swig_obj)) SWIG_fail;
res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 );
if (!SWIG_IsOK(res1)) {
- SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor_IdToPiece" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'");
+ SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor_LoadFromFile" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor *""'");
}
arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1);
- ecode2 = SWIG_AsVal_int(swig_obj[1], &val2);
- if (!SWIG_IsOK(ecode2)) {
- SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "SentencePieceProcessor_IdToPiece" "', argument " "2"" of type '" "int""'");
- }
- arg2 = static_cast< int >(val2);
+ {
+ const PyInputString ustring(swig_obj[1]);
+ if (!ustring.IsAvalable()) {
+ PyErr_SetString(PyExc_TypeError, "not a string");
+ SWIG_fail;
+ }
+ resultobj = ustring.input_type();
+ arg2 = absl::string_view(ustring.data(), ustring.size());
+ }
{
try {
- result = (std::string *) &((sentencepiece::SentencePieceProcessor const *)arg1)->IdToPiece(arg2);
+ result = sentencepiece_SentencePieceProcessor_LoadFromFile(arg1,arg2);
ReleaseResultObject(resultobj);
}
catch (const sentencepiece::util::Status &status) {
}
}
{
- PyObject *input_type = resultobj;
- resultobj = MakePyOutputString(*result, input_type);
+ if (!(&result)->ok()) {
+ SWIG_exception(ToSwigError((&result)->code()), (&result)->ToString().c_str());
+ }
+ resultobj = SWIG_From_bool((&result)->ok());
}
return resultobj;
fail:
}
-SWIGINTERN PyObject *_wrap_SentencePieceProcessor_GetScore(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+SWIGINTERN PyObject *_wrap_SentencePieceProcessor__EncodeAsIds(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
PyObject *resultobj = 0;
sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ;
- int arg2 ;
+ absl::string_view arg2 ;
+ bool arg3 ;
+ int arg4 ;
+ float arg5 ;
+ bool arg6 ;
+ bool arg7 ;
+ bool arg8 ;
+ bool arg9 ;
void *argp1 = 0 ;
int res1 = 0 ;
- int val2 ;
- int ecode2 = 0 ;
- PyObject *swig_obj[2] ;
- float result;
+ bool val3 ;
+ int ecode3 = 0 ;
+ int val4 ;
+ int ecode4 = 0 ;
+ float val5 ;
+ int ecode5 = 0 ;
+ bool val6 ;
+ int ecode6 = 0 ;
+ bool val7 ;
+ int ecode7 = 0 ;
+ bool val8 ;
+ int ecode8 = 0 ;
+ bool val9 ;
+ int ecode9 = 0 ;
+ PyObject *swig_obj[9] ;
+ std::vector< int > result;
- if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor_GetScore", 2, 2, swig_obj)) SWIG_fail;
+ if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor__EncodeAsIds", 9, 9, swig_obj)) SWIG_fail;
res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 );
if (!SWIG_IsOK(res1)) {
- SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor_GetScore" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'");
+ SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor__EncodeAsIds" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'");
}
arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1);
- ecode2 = SWIG_AsVal_int(swig_obj[1], &val2);
- if (!SWIG_IsOK(ecode2)) {
- SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "SentencePieceProcessor_GetScore" "', argument " "2"" of type '" "int""'");
- }
- arg2 = static_cast< int >(val2);
{
- try {
- result = (float)((sentencepiece::SentencePieceProcessor const *)arg1)->GetScore(arg2);
- ReleaseResultObject(resultobj);
- }
- catch (const sentencepiece::util::Status &status) {
- SWIG_exception(ToSwigError(status.code()), status.ToString().c_str());
+ const PyInputString ustring(swig_obj[1]);
+ if (!ustring.IsAvalable()) {
+ PyErr_SetString(PyExc_TypeError, "not a string");
+ SWIG_fail;
}
+ resultobj = ustring.input_type();
+ arg2 = absl::string_view(ustring.data(), ustring.size());
}
- resultobj = SWIG_From_float(static_cast< float >(result));
- return resultobj;
-fail:
- return NULL;
-}
-
-
-SWIGINTERN PyObject *_wrap_SentencePieceProcessor_IsUnknown(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+ ecode3 = SWIG_AsVal_bool(swig_obj[2], &val3);
+ if (!SWIG_IsOK(ecode3)) {
+ SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "SentencePieceProcessor__EncodeAsIds" "', argument " "3"" of type '" "bool""'");
+ }
+ arg3 = static_cast< bool >(val3);
+ ecode4 = SWIG_AsVal_int(swig_obj[3], &val4);
+ if (!SWIG_IsOK(ecode4)) {
+ SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "SentencePieceProcessor__EncodeAsIds" "', argument " "4"" of type '" "int""'");
+ }
+ arg4 = static_cast< int >(val4);
+ ecode5 = SWIG_AsVal_float(swig_obj[4], &val5);
+ if (!SWIG_IsOK(ecode5)) {
+ SWIG_exception_fail(SWIG_ArgError(ecode5), "in method '" "SentencePieceProcessor__EncodeAsIds" "', argument " "5"" of type '" "float""'");
+ }
+ arg5 = static_cast< float >(val5);
+ ecode6 = SWIG_AsVal_bool(swig_obj[5], &val6);
+ if (!SWIG_IsOK(ecode6)) {
+ SWIG_exception_fail(SWIG_ArgError(ecode6), "in method '" "SentencePieceProcessor__EncodeAsIds" "', argument " "6"" of type '" "bool""'");
+ }
+ arg6 = static_cast< bool >(val6);
+ ecode7 = SWIG_AsVal_bool(swig_obj[6], &val7);
+ if (!SWIG_IsOK(ecode7)) {
+ SWIG_exception_fail(SWIG_ArgError(ecode7), "in method '" "SentencePieceProcessor__EncodeAsIds" "', argument " "7"" of type '" "bool""'");
+ }
+ arg7 = static_cast< bool >(val7);
+ ecode8 = SWIG_AsVal_bool(swig_obj[7], &val8);
+ if (!SWIG_IsOK(ecode8)) {
+ SWIG_exception_fail(SWIG_ArgError(ecode8), "in method '" "SentencePieceProcessor__EncodeAsIds" "', argument " "8"" of type '" "bool""'");
+ }
+ arg8 = static_cast< bool >(val8);
+ ecode9 = SWIG_AsVal_bool(swig_obj[8], &val9);
+ if (!SWIG_IsOK(ecode9)) {
+ SWIG_exception_fail(SWIG_ArgError(ecode9), "in method '" "SentencePieceProcessor__EncodeAsIds" "', argument " "9"" of type '" "bool""'");
+ }
+ arg9 = static_cast< bool >(val9);
+ {
+ try {
+ result = sentencepiece_SentencePieceProcessor__EncodeAsIds((sentencepiece::SentencePieceProcessor const *)arg1,arg2,arg3,arg4,arg5,arg6,arg7,arg8,arg9);
+ ReleaseResultObject(resultobj);
+ }
+ catch (const sentencepiece::util::Status &status) {
+ SWIG_exception(ToSwigError(status.code()), status.ToString().c_str());
+ }
+ }
+ {
+ resultobj = PyList_New((&result)->size());
+ for (size_t i = 0; i < (&result)->size(); ++i) {
+ PyList_SetItem(resultobj, i, PyInt_FromLong(static_cast<long>(result[i])));
+ }
+ }
+ return resultobj;
+fail:
+ return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_SentencePieceProcessor__EncodeAsPieces(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
PyObject *resultobj = 0;
sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ;
- int arg2 ;
+ absl::string_view arg2 ;
+ bool arg3 ;
+ int arg4 ;
+ float arg5 ;
+ bool arg6 ;
+ bool arg7 ;
+ bool arg8 ;
+ bool arg9 ;
void *argp1 = 0 ;
int res1 = 0 ;
- int val2 ;
- int ecode2 = 0 ;
- PyObject *swig_obj[2] ;
- bool result;
+ bool val3 ;
+ int ecode3 = 0 ;
+ int val4 ;
+ int ecode4 = 0 ;
+ float val5 ;
+ int ecode5 = 0 ;
+ bool val6 ;
+ int ecode6 = 0 ;
+ bool val7 ;
+ int ecode7 = 0 ;
+ bool val8 ;
+ int ecode8 = 0 ;
+ bool val9 ;
+ int ecode9 = 0 ;
+ PyObject *swig_obj[9] ;
+ std::vector< std::string > result;
- if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor_IsUnknown", 2, 2, swig_obj)) SWIG_fail;
+ if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor__EncodeAsPieces", 9, 9, swig_obj)) SWIG_fail;
res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 );
if (!SWIG_IsOK(res1)) {
- SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor_IsUnknown" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'");
+ SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor__EncodeAsPieces" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'");
}
arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1);
- ecode2 = SWIG_AsVal_int(swig_obj[1], &val2);
- if (!SWIG_IsOK(ecode2)) {
- SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "SentencePieceProcessor_IsUnknown" "', argument " "2"" of type '" "int""'");
+ {
+ const PyInputString ustring(swig_obj[1]);
+ if (!ustring.IsAvalable()) {
+ PyErr_SetString(PyExc_TypeError, "not a string");
+ SWIG_fail;
+ }
+ resultobj = ustring.input_type();
+ arg2 = absl::string_view(ustring.data(), ustring.size());
+ }
+ ecode3 = SWIG_AsVal_bool(swig_obj[2], &val3);
+ if (!SWIG_IsOK(ecode3)) {
+ SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "SentencePieceProcessor__EncodeAsPieces" "', argument " "3"" of type '" "bool""'");
}
- arg2 = static_cast< int >(val2);
+ arg3 = static_cast< bool >(val3);
+ ecode4 = SWIG_AsVal_int(swig_obj[3], &val4);
+ if (!SWIG_IsOK(ecode4)) {
+ SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "SentencePieceProcessor__EncodeAsPieces" "', argument " "4"" of type '" "int""'");
+ }
+ arg4 = static_cast< int >(val4);
+ ecode5 = SWIG_AsVal_float(swig_obj[4], &val5);
+ if (!SWIG_IsOK(ecode5)) {
+ SWIG_exception_fail(SWIG_ArgError(ecode5), "in method '" "SentencePieceProcessor__EncodeAsPieces" "', argument " "5"" of type '" "float""'");
+ }
+ arg5 = static_cast< float >(val5);
+ ecode6 = SWIG_AsVal_bool(swig_obj[5], &val6);
+ if (!SWIG_IsOK(ecode6)) {
+ SWIG_exception_fail(SWIG_ArgError(ecode6), "in method '" "SentencePieceProcessor__EncodeAsPieces" "', argument " "6"" of type '" "bool""'");
+ }
+ arg6 = static_cast< bool >(val6);
+ ecode7 = SWIG_AsVal_bool(swig_obj[6], &val7);
+ if (!SWIG_IsOK(ecode7)) {
+ SWIG_exception_fail(SWIG_ArgError(ecode7), "in method '" "SentencePieceProcessor__EncodeAsPieces" "', argument " "7"" of type '" "bool""'");
+ }
+ arg7 = static_cast< bool >(val7);
+ ecode8 = SWIG_AsVal_bool(swig_obj[7], &val8);
+ if (!SWIG_IsOK(ecode8)) {
+ SWIG_exception_fail(SWIG_ArgError(ecode8), "in method '" "SentencePieceProcessor__EncodeAsPieces" "', argument " "8"" of type '" "bool""'");
+ }
+ arg8 = static_cast< bool >(val8);
+ ecode9 = SWIG_AsVal_bool(swig_obj[8], &val9);
+ if (!SWIG_IsOK(ecode9)) {
+ SWIG_exception_fail(SWIG_ArgError(ecode9), "in method '" "SentencePieceProcessor__EncodeAsPieces" "', argument " "9"" of type '" "bool""'");
+ }
+ arg9 = static_cast< bool >(val9);
{
try {
- result = (bool)((sentencepiece::SentencePieceProcessor const *)arg1)->IsUnknown(arg2);
+ result = sentencepiece_SentencePieceProcessor__EncodeAsPieces((sentencepiece::SentencePieceProcessor const *)arg1,arg2,arg3,arg4,arg5,arg6,arg7,arg8,arg9);
ReleaseResultObject(resultobj);
}
catch (const sentencepiece::util::Status &status) {
SWIG_exception(ToSwigError(status.code()), status.ToString().c_str());
}
}
- resultobj = SWIG_From_bool(static_cast< bool >(result));
+ {
+ PyObject *input_type = resultobj;
+ resultobj = PyList_New((&result)->size());
+ for (size_t i = 0; i < (&result)->size(); ++i) {
+ PyList_SetItem(resultobj, i, MakePyOutputString(result[i], input_type));
+ }
+ }
return resultobj;
fail:
return NULL;
}
-SWIGINTERN PyObject *_wrap_SentencePieceProcessor_IsControl(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+SWIGINTERN PyObject *_wrap_SentencePieceProcessor__EncodeAsSerializedProto(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
PyObject *resultobj = 0;
sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ;
- int arg2 ;
+ absl::string_view arg2 ;
+ bool arg3 ;
+ int arg4 ;
+ float arg5 ;
+ bool arg6 ;
+ bool arg7 ;
+ bool arg8 ;
+ bool arg9 ;
void *argp1 = 0 ;
int res1 = 0 ;
- int val2 ;
- int ecode2 = 0 ;
- PyObject *swig_obj[2] ;
- bool result;
+ bool val3 ;
+ int ecode3 = 0 ;
+ int val4 ;
+ int ecode4 = 0 ;
+ float val5 ;
+ int ecode5 = 0 ;
+ bool val6 ;
+ int ecode6 = 0 ;
+ bool val7 ;
+ int ecode7 = 0 ;
+ bool val8 ;
+ int ecode8 = 0 ;
+ bool val9 ;
+ int ecode9 = 0 ;
+ PyObject *swig_obj[9] ;
+ sentencepiece::util::bytes result;
- if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor_IsControl", 2, 2, swig_obj)) SWIG_fail;
+ if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor__EncodeAsSerializedProto", 9, 9, swig_obj)) SWIG_fail;
res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 );
if (!SWIG_IsOK(res1)) {
- SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor_IsControl" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'");
+ SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor__EncodeAsSerializedProto" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'");
}
arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1);
- ecode2 = SWIG_AsVal_int(swig_obj[1], &val2);
- if (!SWIG_IsOK(ecode2)) {
- SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "SentencePieceProcessor_IsControl" "', argument " "2"" of type '" "int""'");
+ {
+ const PyInputString ustring(swig_obj[1]);
+ if (!ustring.IsAvalable()) {
+ PyErr_SetString(PyExc_TypeError, "not a string");
+ SWIG_fail;
+ }
+ resultobj = ustring.input_type();
+ arg2 = absl::string_view(ustring.data(), ustring.size());
+ }
+ ecode3 = SWIG_AsVal_bool(swig_obj[2], &val3);
+ if (!SWIG_IsOK(ecode3)) {
+ SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "SentencePieceProcessor__EncodeAsSerializedProto" "', argument " "3"" of type '" "bool""'");
}
- arg2 = static_cast< int >(val2);
+ arg3 = static_cast< bool >(val3);
+ ecode4 = SWIG_AsVal_int(swig_obj[3], &val4);
+ if (!SWIG_IsOK(ecode4)) {
+ SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "SentencePieceProcessor__EncodeAsSerializedProto" "', argument " "4"" of type '" "int""'");
+ }
+ arg4 = static_cast< int >(val4);
+ ecode5 = SWIG_AsVal_float(swig_obj[4], &val5);
+ if (!SWIG_IsOK(ecode5)) {
+ SWIG_exception_fail(SWIG_ArgError(ecode5), "in method '" "SentencePieceProcessor__EncodeAsSerializedProto" "', argument " "5"" of type '" "float""'");
+ }
+ arg5 = static_cast< float >(val5);
+ ecode6 = SWIG_AsVal_bool(swig_obj[5], &val6);
+ if (!SWIG_IsOK(ecode6)) {
+ SWIG_exception_fail(SWIG_ArgError(ecode6), "in method '" "SentencePieceProcessor__EncodeAsSerializedProto" "', argument " "6"" of type '" "bool""'");
+ }
+ arg6 = static_cast< bool >(val6);
+ ecode7 = SWIG_AsVal_bool(swig_obj[6], &val7);
+ if (!SWIG_IsOK(ecode7)) {
+ SWIG_exception_fail(SWIG_ArgError(ecode7), "in method '" "SentencePieceProcessor__EncodeAsSerializedProto" "', argument " "7"" of type '" "bool""'");
+ }
+ arg7 = static_cast< bool >(val7);
+ ecode8 = SWIG_AsVal_bool(swig_obj[7], &val8);
+ if (!SWIG_IsOK(ecode8)) {
+ SWIG_exception_fail(SWIG_ArgError(ecode8), "in method '" "SentencePieceProcessor__EncodeAsSerializedProto" "', argument " "8"" of type '" "bool""'");
+ }
+ arg8 = static_cast< bool >(val8);
+ ecode9 = SWIG_AsVal_bool(swig_obj[8], &val9);
+ if (!SWIG_IsOK(ecode9)) {
+ SWIG_exception_fail(SWIG_ArgError(ecode9), "in method '" "SentencePieceProcessor__EncodeAsSerializedProto" "', argument " "9"" of type '" "bool""'");
+ }
+ arg9 = static_cast< bool >(val9);
{
try {
- result = (bool)((sentencepiece::SentencePieceProcessor const *)arg1)->IsControl(arg2);
+ result = sentencepiece_SentencePieceProcessor__EncodeAsSerializedProto((sentencepiece::SentencePieceProcessor const *)arg1,arg2,arg3,arg4,arg5,arg6,arg7,arg8,arg9);
ReleaseResultObject(resultobj);
}
catch (const sentencepiece::util::Status &status) {
SWIG_exception(ToSwigError(status.code()), status.ToString().c_str());
}
}
- resultobj = SWIG_From_bool(static_cast< bool >(result));
+ {
+ resultobj = MakePyOutputBytes(result);
+ }
return resultobj;
fail:
return NULL;
}
-SWIGINTERN PyObject *_wrap_SentencePieceProcessor_IsUnused(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+SWIGINTERN PyObject *_wrap_SentencePieceProcessor__EncodeAsIdsBatch(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
PyObject *resultobj = 0;
sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ;
- int arg2 ;
+ std::vector< absl::string_view > *arg2 = 0 ;
+ int arg3 ;
+ bool arg4 ;
+ int arg5 ;
+ float arg6 ;
+ bool arg7 ;
+ bool arg8 ;
+ bool arg9 ;
+ bool arg10 ;
void *argp1 = 0 ;
int res1 = 0 ;
- int val2 ;
- int ecode2 = 0 ;
- PyObject *swig_obj[2] ;
- bool result;
+ int val3 ;
+ int ecode3 = 0 ;
+ bool val4 ;
+ int ecode4 = 0 ;
+ int val5 ;
+ int ecode5 = 0 ;
+ float val6 ;
+ int ecode6 = 0 ;
+ bool val7 ;
+ int ecode7 = 0 ;
+ bool val8 ;
+ int ecode8 = 0 ;
+ bool val9 ;
+ int ecode9 = 0 ;
+ bool val10 ;
+ int ecode10 = 0 ;
+ PyObject *swig_obj[10] ;
+ std::vector< std::vector< int > > result;
- if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor_IsUnused", 2, 2, swig_obj)) SWIG_fail;
+ if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor__EncodeAsIdsBatch", 10, 10, swig_obj)) SWIG_fail;
res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 );
if (!SWIG_IsOK(res1)) {
- SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor_IsUnused" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'");
+ SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor__EncodeAsIdsBatch" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'");
}
arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1);
- ecode2 = SWIG_AsVal_int(swig_obj[1], &val2);
- if (!SWIG_IsOK(ecode2)) {
- SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "SentencePieceProcessor_IsUnused" "', argument " "2"" of type '" "int""'");
+ {
+ std::vector<absl::string_view> *out = nullptr;
+ if (PyList_Check(swig_obj[1])) {
+ const size_t size = PyList_Size(swig_obj[1]);
+ out = new std::vector<absl::string_view>(size);
+ for (size_t i = 0; i < size; ++i) {
+ const PyInputString ustring(PyList_GetItem(swig_obj[1], i));
+ if (ustring.IsAvalable()) {
+ (*out)[i] = absl::string_view(ustring.data(), ustring.size());
+ } else {
+ PyErr_SetString(PyExc_TypeError, "list must contain strings");
+ SWIG_fail;
+ }
+ resultobj = ustring.input_type();
+ }
+ } else {
+ PyErr_SetString(PyExc_TypeError, "not a list");
+ SWIG_fail;
+ }
+ arg2 = out;
+ }
+ ecode3 = SWIG_AsVal_int(swig_obj[2], &val3);
+ if (!SWIG_IsOK(ecode3)) {
+ SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "SentencePieceProcessor__EncodeAsIdsBatch" "', argument " "3"" of type '" "int""'");
}
- arg2 = static_cast< int >(val2);
+ arg3 = static_cast< int >(val3);
+ ecode4 = SWIG_AsVal_bool(swig_obj[3], &val4);
+ if (!SWIG_IsOK(ecode4)) {
+ SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "SentencePieceProcessor__EncodeAsIdsBatch" "', argument " "4"" of type '" "bool""'");
+ }
+ arg4 = static_cast< bool >(val4);
+ ecode5 = SWIG_AsVal_int(swig_obj[4], &val5);
+ if (!SWIG_IsOK(ecode5)) {
+ SWIG_exception_fail(SWIG_ArgError(ecode5), "in method '" "SentencePieceProcessor__EncodeAsIdsBatch" "', argument " "5"" of type '" "int""'");
+ }
+ arg5 = static_cast< int >(val5);
+ ecode6 = SWIG_AsVal_float(swig_obj[5], &val6);
+ if (!SWIG_IsOK(ecode6)) {
+ SWIG_exception_fail(SWIG_ArgError(ecode6), "in method '" "SentencePieceProcessor__EncodeAsIdsBatch" "', argument " "6"" of type '" "float""'");
+ }
+ arg6 = static_cast< float >(val6);
+ ecode7 = SWIG_AsVal_bool(swig_obj[6], &val7);
+ if (!SWIG_IsOK(ecode7)) {
+ SWIG_exception_fail(SWIG_ArgError(ecode7), "in method '" "SentencePieceProcessor__EncodeAsIdsBatch" "', argument " "7"" of type '" "bool""'");
+ }
+ arg7 = static_cast< bool >(val7);
+ ecode8 = SWIG_AsVal_bool(swig_obj[7], &val8);
+ if (!SWIG_IsOK(ecode8)) {
+ SWIG_exception_fail(SWIG_ArgError(ecode8), "in method '" "SentencePieceProcessor__EncodeAsIdsBatch" "', argument " "8"" of type '" "bool""'");
+ }
+ arg8 = static_cast< bool >(val8);
+ ecode9 = SWIG_AsVal_bool(swig_obj[8], &val9);
+ if (!SWIG_IsOK(ecode9)) {
+ SWIG_exception_fail(SWIG_ArgError(ecode9), "in method '" "SentencePieceProcessor__EncodeAsIdsBatch" "', argument " "9"" of type '" "bool""'");
+ }
+ arg9 = static_cast< bool >(val9);
+ ecode10 = SWIG_AsVal_bool(swig_obj[9], &val10);
+ if (!SWIG_IsOK(ecode10)) {
+ SWIG_exception_fail(SWIG_ArgError(ecode10), "in method '" "SentencePieceProcessor__EncodeAsIdsBatch" "', argument " "10"" of type '" "bool""'");
+ }
+ arg10 = static_cast< bool >(val10);
{
try {
- result = (bool)((sentencepiece::SentencePieceProcessor const *)arg1)->IsUnused(arg2);
+ result = sentencepiece_SentencePieceProcessor__EncodeAsIdsBatch((sentencepiece::SentencePieceProcessor const *)arg1,(std::vector< absl::string_view > const &)*arg2,arg3,arg4,arg5,arg6,arg7,arg8,arg9,arg10);
ReleaseResultObject(resultobj);
}
catch (const sentencepiece::util::Status &status) {
SWIG_exception(ToSwigError(status.code()), status.ToString().c_str());
}
}
- resultobj = SWIG_From_bool(static_cast< bool >(result));
+ {
+ resultobj = PyList_New((&result)->size());
+ for (size_t i = 0; i < (&result)->size(); ++i) {
+ PyObject *obj = PyList_New(result[i].size());
+ for (size_t j = 0; j < result[i].size(); ++j) {
+ PyList_SetItem(obj, j, PyInt_FromLong(static_cast<long>(result[i][j])));
+ }
+ PyList_SetItem(resultobj, i, obj);
+ }
+ }
+ {
+ delete arg2;
+ }
return resultobj;
fail:
+ {
+ delete arg2;
+ }
return NULL;
}
-SWIGINTERN PyObject *_wrap_SentencePieceProcessor_IsByte(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+SWIGINTERN PyObject *_wrap_SentencePieceProcessor__EncodeAsPiecesBatch(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
PyObject *resultobj = 0;
sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ;
- int arg2 ;
+ std::vector< absl::string_view > *arg2 = 0 ;
+ int arg3 ;
+ bool arg4 ;
+ int arg5 ;
+ float arg6 ;
+ bool arg7 ;
+ bool arg8 ;
+ bool arg9 ;
+ bool arg10 ;
void *argp1 = 0 ;
int res1 = 0 ;
- int val2 ;
- int ecode2 = 0 ;
- PyObject *swig_obj[2] ;
- bool result;
+ int val3 ;
+ int ecode3 = 0 ;
+ bool val4 ;
+ int ecode4 = 0 ;
+ int val5 ;
+ int ecode5 = 0 ;
+ float val6 ;
+ int ecode6 = 0 ;
+ bool val7 ;
+ int ecode7 = 0 ;
+ bool val8 ;
+ int ecode8 = 0 ;
+ bool val9 ;
+ int ecode9 = 0 ;
+ bool val10 ;
+ int ecode10 = 0 ;
+ PyObject *swig_obj[10] ;
+ std::vector< std::vector< std::string > > result;
- if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor_IsByte", 2, 2, swig_obj)) SWIG_fail;
+ if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor__EncodeAsPiecesBatch", 10, 10, swig_obj)) SWIG_fail;
res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 );
if (!SWIG_IsOK(res1)) {
- SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor_IsByte" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'");
+ SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor__EncodeAsPiecesBatch" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'");
}
arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1);
- ecode2 = SWIG_AsVal_int(swig_obj[1], &val2);
- if (!SWIG_IsOK(ecode2)) {
- SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "SentencePieceProcessor_IsByte" "', argument " "2"" of type '" "int""'");
+ {
+ std::vector<absl::string_view> *out = nullptr;
+ if (PyList_Check(swig_obj[1])) {
+ const size_t size = PyList_Size(swig_obj[1]);
+ out = new std::vector<absl::string_view>(size);
+ for (size_t i = 0; i < size; ++i) {
+ const PyInputString ustring(PyList_GetItem(swig_obj[1], i));
+ if (ustring.IsAvalable()) {
+ (*out)[i] = absl::string_view(ustring.data(), ustring.size());
+ } else {
+ PyErr_SetString(PyExc_TypeError, "list must contain strings");
+ SWIG_fail;
+ }
+ resultobj = ustring.input_type();
+ }
+ } else {
+ PyErr_SetString(PyExc_TypeError, "not a list");
+ SWIG_fail;
+ }
+ arg2 = out;
+ }
+ ecode3 = SWIG_AsVal_int(swig_obj[2], &val3);
+ if (!SWIG_IsOK(ecode3)) {
+ SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "SentencePieceProcessor__EncodeAsPiecesBatch" "', argument " "3"" of type '" "int""'");
}
- arg2 = static_cast< int >(val2);
+ arg3 = static_cast< int >(val3);
+ ecode4 = SWIG_AsVal_bool(swig_obj[3], &val4);
+ if (!SWIG_IsOK(ecode4)) {
+ SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "SentencePieceProcessor__EncodeAsPiecesBatch" "', argument " "4"" of type '" "bool""'");
+ }
+ arg4 = static_cast< bool >(val4);
+ ecode5 = SWIG_AsVal_int(swig_obj[4], &val5);
+ if (!SWIG_IsOK(ecode5)) {
+ SWIG_exception_fail(SWIG_ArgError(ecode5), "in method '" "SentencePieceProcessor__EncodeAsPiecesBatch" "', argument " "5"" of type '" "int""'");
+ }
+ arg5 = static_cast< int >(val5);
+ ecode6 = SWIG_AsVal_float(swig_obj[5], &val6);
+ if (!SWIG_IsOK(ecode6)) {
+ SWIG_exception_fail(SWIG_ArgError(ecode6), "in method '" "SentencePieceProcessor__EncodeAsPiecesBatch" "', argument " "6"" of type '" "float""'");
+ }
+ arg6 = static_cast< float >(val6);
+ ecode7 = SWIG_AsVal_bool(swig_obj[6], &val7);
+ if (!SWIG_IsOK(ecode7)) {
+ SWIG_exception_fail(SWIG_ArgError(ecode7), "in method '" "SentencePieceProcessor__EncodeAsPiecesBatch" "', argument " "7"" of type '" "bool""'");
+ }
+ arg7 = static_cast< bool >(val7);
+ ecode8 = SWIG_AsVal_bool(swig_obj[7], &val8);
+ if (!SWIG_IsOK(ecode8)) {
+ SWIG_exception_fail(SWIG_ArgError(ecode8), "in method '" "SentencePieceProcessor__EncodeAsPiecesBatch" "', argument " "8"" of type '" "bool""'");
+ }
+ arg8 = static_cast< bool >(val8);
+ ecode9 = SWIG_AsVal_bool(swig_obj[8], &val9);
+ if (!SWIG_IsOK(ecode9)) {
+ SWIG_exception_fail(SWIG_ArgError(ecode9), "in method '" "SentencePieceProcessor__EncodeAsPiecesBatch" "', argument " "9"" of type '" "bool""'");
+ }
+ arg9 = static_cast< bool >(val9);
+ ecode10 = SWIG_AsVal_bool(swig_obj[9], &val10);
+ if (!SWIG_IsOK(ecode10)) {
+ SWIG_exception_fail(SWIG_ArgError(ecode10), "in method '" "SentencePieceProcessor__EncodeAsPiecesBatch" "', argument " "10"" of type '" "bool""'");
+ }
+ arg10 = static_cast< bool >(val10);
{
try {
- result = (bool)((sentencepiece::SentencePieceProcessor const *)arg1)->IsByte(arg2);
+ result = sentencepiece_SentencePieceProcessor__EncodeAsPiecesBatch((sentencepiece::SentencePieceProcessor const *)arg1,(std::vector< absl::string_view > const &)*arg2,arg3,arg4,arg5,arg6,arg7,arg8,arg9,arg10);
ReleaseResultObject(resultobj);
}
catch (const sentencepiece::util::Status &status) {
SWIG_exception(ToSwigError(status.code()), status.ToString().c_str());
}
}
- resultobj = SWIG_From_bool(static_cast< bool >(result));
+ {
+ PyObject *input_type = resultobj;
+ resultobj = PyList_New((&result)->size());
+ for (size_t i = 0; i < (&result)->size(); ++i) {
+ PyObject *obj = PyList_New(result[i].size());
+ for (size_t j = 0; j < result[i].size(); ++j) {
+ PyList_SetItem(obj, j, MakePyOutputString(result[i][j], input_type));
+ }
+ PyList_SetItem(resultobj, i, obj);
+ }
+ }
+ {
+ delete arg2;
+ }
return resultobj;
fail:
+ {
+ delete arg2;
+ }
return NULL;
}
-SWIGINTERN PyObject *_wrap_SentencePieceProcessor_unk_id(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+SWIGINTERN PyObject *_wrap_SentencePieceProcessor__EncodeAsSerializedProtoBatch(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
PyObject *resultobj = 0;
sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ;
+ std::vector< absl::string_view > *arg2 = 0 ;
+ int arg3 ;
+ bool arg4 ;
+ int arg5 ;
+ float arg6 ;
+ bool arg7 ;
+ bool arg8 ;
+ bool arg9 ;
+ bool arg10 ;
void *argp1 = 0 ;
int res1 = 0 ;
- PyObject *swig_obj[1] ;
- int result;
+ int val3 ;
+ int ecode3 = 0 ;
+ bool val4 ;
+ int ecode4 = 0 ;
+ int val5 ;
+ int ecode5 = 0 ;
+ float val6 ;
+ int ecode6 = 0 ;
+ bool val7 ;
+ int ecode7 = 0 ;
+ bool val8 ;
+ int ecode8 = 0 ;
+ bool val9 ;
+ int ecode9 = 0 ;
+ bool val10 ;
+ int ecode10 = 0 ;
+ PyObject *swig_obj[10] ;
+ BytesArray result;
- if (!args) SWIG_fail;
- swig_obj[0] = args;
+ if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor__EncodeAsSerializedProtoBatch", 10, 10, swig_obj)) SWIG_fail;
res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 );
if (!SWIG_IsOK(res1)) {
- SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor_unk_id" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'");
+ SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor__EncodeAsSerializedProtoBatch" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'");
}
arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1);
+ {
+ std::vector<absl::string_view> *out = nullptr;
+ if (PyList_Check(swig_obj[1])) {
+ const size_t size = PyList_Size(swig_obj[1]);
+ out = new std::vector<absl::string_view>(size);
+ for (size_t i = 0; i < size; ++i) {
+ const PyInputString ustring(PyList_GetItem(swig_obj[1], i));
+ if (ustring.IsAvalable()) {
+ (*out)[i] = absl::string_view(ustring.data(), ustring.size());
+ } else {
+ PyErr_SetString(PyExc_TypeError, "list must contain strings");
+ SWIG_fail;
+ }
+ resultobj = ustring.input_type();
+ }
+ } else {
+ PyErr_SetString(PyExc_TypeError, "not a list");
+ SWIG_fail;
+ }
+ arg2 = out;
+ }
+ ecode3 = SWIG_AsVal_int(swig_obj[2], &val3);
+ if (!SWIG_IsOK(ecode3)) {
+ SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "SentencePieceProcessor__EncodeAsSerializedProtoBatch" "', argument " "3"" of type '" "int""'");
+ }
+ arg3 = static_cast< int >(val3);
+ ecode4 = SWIG_AsVal_bool(swig_obj[3], &val4);
+ if (!SWIG_IsOK(ecode4)) {
+ SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "SentencePieceProcessor__EncodeAsSerializedProtoBatch" "', argument " "4"" of type '" "bool""'");
+ }
+ arg4 = static_cast< bool >(val4);
+ ecode5 = SWIG_AsVal_int(swig_obj[4], &val5);
+ if (!SWIG_IsOK(ecode5)) {
+ SWIG_exception_fail(SWIG_ArgError(ecode5), "in method '" "SentencePieceProcessor__EncodeAsSerializedProtoBatch" "', argument " "5"" of type '" "int""'");
+ }
+ arg5 = static_cast< int >(val5);
+ ecode6 = SWIG_AsVal_float(swig_obj[5], &val6);
+ if (!SWIG_IsOK(ecode6)) {
+ SWIG_exception_fail(SWIG_ArgError(ecode6), "in method '" "SentencePieceProcessor__EncodeAsSerializedProtoBatch" "', argument " "6"" of type '" "float""'");
+ }
+ arg6 = static_cast< float >(val6);
+ ecode7 = SWIG_AsVal_bool(swig_obj[6], &val7);
+ if (!SWIG_IsOK(ecode7)) {
+ SWIG_exception_fail(SWIG_ArgError(ecode7), "in method '" "SentencePieceProcessor__EncodeAsSerializedProtoBatch" "', argument " "7"" of type '" "bool""'");
+ }
+ arg7 = static_cast< bool >(val7);
+ ecode8 = SWIG_AsVal_bool(swig_obj[7], &val8);
+ if (!SWIG_IsOK(ecode8)) {
+ SWIG_exception_fail(SWIG_ArgError(ecode8), "in method '" "SentencePieceProcessor__EncodeAsSerializedProtoBatch" "', argument " "8"" of type '" "bool""'");
+ }
+ arg8 = static_cast< bool >(val8);
+ ecode9 = SWIG_AsVal_bool(swig_obj[8], &val9);
+ if (!SWIG_IsOK(ecode9)) {
+ SWIG_exception_fail(SWIG_ArgError(ecode9), "in method '" "SentencePieceProcessor__EncodeAsSerializedProtoBatch" "', argument " "9"" of type '" "bool""'");
+ }
+ arg9 = static_cast< bool >(val9);
+ ecode10 = SWIG_AsVal_bool(swig_obj[9], &val10);
+ if (!SWIG_IsOK(ecode10)) {
+ SWIG_exception_fail(SWIG_ArgError(ecode10), "in method '" "SentencePieceProcessor__EncodeAsSerializedProtoBatch" "', argument " "10"" of type '" "bool""'");
+ }
+ arg10 = static_cast< bool >(val10);
{
try {
- result = (int)((sentencepiece::SentencePieceProcessor const *)arg1)->unk_id();
+ result = sentencepiece_SentencePieceProcessor__EncodeAsSerializedProtoBatch((sentencepiece::SentencePieceProcessor const *)arg1,(std::vector< absl::string_view > const &)*arg2,arg3,arg4,arg5,arg6,arg7,arg8,arg9,arg10);
ReleaseResultObject(resultobj);
}
catch (const sentencepiece::util::Status &status) {
SWIG_exception(ToSwigError(status.code()), status.ToString().c_str());
}
}
- resultobj = SWIG_From_int(static_cast< int >(result));
+ {
+ resultobj = PyList_New((&result)->size());
+ for (size_t i = 0; i < (&result)->size(); ++i) {
+ PyList_SetItem(resultobj, i, MakePyOutputBytes(result[i]));
+ }
+ }
+ {
+ delete arg2;
+ }
return resultobj;
fail:
+ {
+ delete arg2;
+ }
return NULL;
}
-SWIGINTERN PyObject *_wrap_SentencePieceProcessor_bos_id(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+SWIGINTERN PyObject *_wrap_SentencePieceProcessor__DecodeIds(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
PyObject *resultobj = 0;
sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ;
+ std::vector< int > *arg2 = 0 ;
void *argp1 = 0 ;
int res1 = 0 ;
- PyObject *swig_obj[1] ;
- int result;
+ PyObject *swig_obj[2] ;
+ std::string result;
- if (!args) SWIG_fail;
- swig_obj[0] = args;
+ if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor__DecodeIds", 2, 2, swig_obj)) SWIG_fail;
res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 );
if (!SWIG_IsOK(res1)) {
- SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor_bos_id" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'");
+ SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor__DecodeIds" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'");
}
arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1);
+ {
+ std::vector<int> *out = nullptr;
+ if (PyList_Check(swig_obj[1])) {
+ const size_t size = PyList_Size(swig_obj[1]);
+ out = new std::vector<int>(size);
+ for (size_t i = 0; i < size; ++i) {
+ PyObject *o = PyList_GetItem(swig_obj[1], i);
+ if (PyInt_Check(o)) {
+ (*out)[i] = static_cast<int>(PyInt_AsLong(o));
+ } else {
+ PyErr_SetString(PyExc_TypeError,"list must contain integers");
+ SWIG_fail;
+ }
+ }
+ } else {
+ PyErr_SetString(PyExc_TypeError,"not a list");
+ SWIG_fail;
+ }
+ arg2 = out;
+ }
{
try {
- result = (int)((sentencepiece::SentencePieceProcessor const *)arg1)->bos_id();
+ result = sentencepiece_SentencePieceProcessor__DecodeIds((sentencepiece::SentencePieceProcessor const *)arg1,(std::vector< int > const &)*arg2);
ReleaseResultObject(resultobj);
}
catch (const sentencepiece::util::Status &status) {
SWIG_exception(ToSwigError(status.code()), status.ToString().c_str());
}
}
- resultobj = SWIG_From_int(static_cast< int >(result));
+ {
+ PyObject *input_type = resultobj;
+ resultobj = MakePyOutputString(result, input_type);
+ }
+ {
+ delete arg2;
+ }
return resultobj;
fail:
+ {
+ delete arg2;
+ }
return NULL;
}
-SWIGINTERN PyObject *_wrap_SentencePieceProcessor_eos_id(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+SWIGINTERN PyObject *_wrap_SentencePieceProcessor__DecodePieces(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
PyObject *resultobj = 0;
sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ;
+ std::vector< std::string > *arg2 = 0 ;
void *argp1 = 0 ;
int res1 = 0 ;
- PyObject *swig_obj[1] ;
- int result;
+ PyObject *swig_obj[2] ;
+ std::string result;
- if (!args) SWIG_fail;
- swig_obj[0] = args;
+ if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor__DecodePieces", 2, 2, swig_obj)) SWIG_fail;
res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 );
if (!SWIG_IsOK(res1)) {
- SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor_eos_id" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'");
+ SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor__DecodePieces" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'");
+ }
+ arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1);
+ {
+ std::vector<std::string> *out = nullptr;
+ if (PyList_Check(swig_obj[1])) {
+ const size_t size = PyList_Size(swig_obj[1]);
+ out = new std::vector<std::string>(size);
+ for (size_t i = 0; i < size; ++i) {
+ const PyInputString ustring(PyList_GetItem(swig_obj[1], i));
+ if (ustring.IsAvalable()) {
+ (*out)[i].assign(ustring.data(), ustring.size());
+ } else {
+ PyErr_SetString(PyExc_TypeError, "list must contain strings");
+ SWIG_fail;
+ }
+ resultobj = ustring.input_type();
+ }
+ } else {
+ PyErr_SetString(PyExc_TypeError, "not a list");
+ SWIG_fail;
+ }
+ arg2 = out;
}
- arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1);
{
try {
- result = (int)((sentencepiece::SentencePieceProcessor const *)arg1)->eos_id();
+ result = sentencepiece_SentencePieceProcessor__DecodePieces((sentencepiece::SentencePieceProcessor const *)arg1,(std::vector< std::string > const &)*arg2);
ReleaseResultObject(resultobj);
}
catch (const sentencepiece::util::Status &status) {
SWIG_exception(ToSwigError(status.code()), status.ToString().c_str());
}
}
- resultobj = SWIG_From_int(static_cast< int >(result));
+ {
+ PyObject *input_type = resultobj;
+ resultobj = MakePyOutputString(result, input_type);
+ }
+ {
+ delete arg2;
+ }
return resultobj;
fail:
+ {
+ delete arg2;
+ }
return NULL;
}
-SWIGINTERN PyObject *_wrap_SentencePieceProcessor_pad_id(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+SWIGINTERN PyObject *_wrap_SentencePieceProcessor__DecodeIdsAsSerializedProto(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
PyObject *resultobj = 0;
sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ;
+ std::vector< int > *arg2 = 0 ;
void *argp1 = 0 ;
int res1 = 0 ;
- PyObject *swig_obj[1] ;
- int result;
+ PyObject *swig_obj[2] ;
+ sentencepiece::util::bytes result;
- if (!args) SWIG_fail;
- swig_obj[0] = args;
+ if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor__DecodeIdsAsSerializedProto", 2, 2, swig_obj)) SWIG_fail;
res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 );
if (!SWIG_IsOK(res1)) {
- SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor_pad_id" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'");
+ SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor__DecodeIdsAsSerializedProto" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'");
}
arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1);
+ {
+ std::vector<int> *out = nullptr;
+ if (PyList_Check(swig_obj[1])) {
+ const size_t size = PyList_Size(swig_obj[1]);
+ out = new std::vector<int>(size);
+ for (size_t i = 0; i < size; ++i) {
+ PyObject *o = PyList_GetItem(swig_obj[1], i);
+ if (PyInt_Check(o)) {
+ (*out)[i] = static_cast<int>(PyInt_AsLong(o));
+ } else {
+ PyErr_SetString(PyExc_TypeError,"list must contain integers");
+ SWIG_fail;
+ }
+ }
+ } else {
+ PyErr_SetString(PyExc_TypeError,"not a list");
+ SWIG_fail;
+ }
+ arg2 = out;
+ }
{
try {
- result = (int)((sentencepiece::SentencePieceProcessor const *)arg1)->pad_id();
+ result = sentencepiece_SentencePieceProcessor__DecodeIdsAsSerializedProto((sentencepiece::SentencePieceProcessor const *)arg1,(std::vector< int > const &)*arg2);
ReleaseResultObject(resultobj);
}
catch (const sentencepiece::util::Status &status) {
SWIG_exception(ToSwigError(status.code()), status.ToString().c_str());
}
}
- resultobj = SWIG_From_int(static_cast< int >(result));
+ {
+ resultobj = MakePyOutputBytes(result);
+ }
+ {
+ delete arg2;
+ }
return resultobj;
fail:
+ {
+ delete arg2;
+ }
return NULL;
}
-SWIGINTERN PyObject *_wrap_SentencePieceProcessor_serialized_model_proto(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+SWIGINTERN PyObject *_wrap_SentencePieceProcessor__DecodePiecesAsSerializedProto(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
PyObject *resultobj = 0;
sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ;
+ std::vector< std::string > *arg2 = 0 ;
void *argp1 = 0 ;
int res1 = 0 ;
- PyObject *swig_obj[1] ;
+ PyObject *swig_obj[2] ;
sentencepiece::util::bytes result;
- if (!args) SWIG_fail;
- swig_obj[0] = args;
+ if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor__DecodePiecesAsSerializedProto", 2, 2, swig_obj)) SWIG_fail;
res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 );
if (!SWIG_IsOK(res1)) {
- SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor_serialized_model_proto" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'");
+ SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor__DecodePiecesAsSerializedProto" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'");
}
arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1);
+ {
+ std::vector<std::string> *out = nullptr;
+ if (PyList_Check(swig_obj[1])) {
+ const size_t size = PyList_Size(swig_obj[1]);
+ out = new std::vector<std::string>(size);
+ for (size_t i = 0; i < size; ++i) {
+ const PyInputString ustring(PyList_GetItem(swig_obj[1], i));
+ if (ustring.IsAvalable()) {
+ (*out)[i].assign(ustring.data(), ustring.size());
+ } else {
+ PyErr_SetString(PyExc_TypeError, "list must contain strings");
+ SWIG_fail;
+ }
+ resultobj = ustring.input_type();
+ }
+ } else {
+ PyErr_SetString(PyExc_TypeError, "not a list");
+ SWIG_fail;
+ }
+ arg2 = out;
+ }
{
try {
- result = ((sentencepiece::SentencePieceProcessor const *)arg1)->serialized_model_proto();
+ result = sentencepiece_SentencePieceProcessor__DecodePiecesAsSerializedProto((sentencepiece::SentencePieceProcessor const *)arg1,(std::vector< std::string > const &)*arg2);
ReleaseResultObject(resultobj);
}
catch (const sentencepiece::util::Status &status) {
{
resultobj = MakePyOutputBytes(result);
}
+ {
+ delete arg2;
+ }
return resultobj;
fail:
+ {
+ delete arg2;
+ }
return NULL;
}
-SWIGINTERN PyObject *_wrap_SentencePieceProcessor_LoadFromFile(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+SWIGINTERN PyObject *_wrap_SentencePieceProcessor__DecodeIdsBatch(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
PyObject *resultobj = 0;
sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ;
- absl::string_view arg2 ;
+ std::vector< std::vector< int > > *arg2 = 0 ;
+ int arg3 ;
void *argp1 = 0 ;
int res1 = 0 ;
- PyObject *swig_obj[2] ;
- sentencepiece::util::Status result;
+ int val3 ;
+ int ecode3 = 0 ;
+ PyObject *swig_obj[3] ;
+ std::vector< std::string > result;
- if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor_LoadFromFile", 2, 2, swig_obj)) SWIG_fail;
+ if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor__DecodeIdsBatch", 3, 3, swig_obj)) SWIG_fail;
res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 );
if (!SWIG_IsOK(res1)) {
- SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor_LoadFromFile" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor *""'");
+ SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor__DecodeIdsBatch" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'");
}
arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1);
{
- const PyInputString ustring(swig_obj[1]);
- if (!ustring.IsAvalable()) {
- PyErr_SetString(PyExc_TypeError, "not a string");
+ std::vector<std::vector<int>> *out = nullptr;
+ if (PyList_Check(swig_obj[1])) {
+ const size_t size = PyList_Size(swig_obj[1]);
+ out = new std::vector<std::vector<int>>(size);
+ for (size_t i = 0; i < size; ++i) {
+ PyObject *o = PyList_GetItem(swig_obj[1], i);
+ if (PyList_Check(o)) {
+ const size_t size2 = PyList_Size(o);
+ (*out)[i].resize(size2);
+ for (size_t j = 0; j < size2; ++j) {
+ PyObject *o2 = PyList_GetItem(o, j);
+ if (PyInt_Check(o2)) {
+ (*out)[i][j] = static_cast<int>(PyInt_AsLong(o2));
+ } else {
+ PyErr_SetString(PyExc_TypeError, "list must contain strings");
+ SWIG_fail;
+ }
+ }
+ } else {
+ PyErr_SetString(PyExc_TypeError, "not a list");
+ SWIG_fail;
+ }
+ }
+ } else {
+ PyErr_SetString(PyExc_TypeError,"not a list");
SWIG_fail;
}
- resultobj = ustring.input_type();
- arg2 = absl::string_view(ustring.data(), ustring.size());
+ arg2 = out;
}
+ ecode3 = SWIG_AsVal_int(swig_obj[2], &val3);
+ if (!SWIG_IsOK(ecode3)) {
+ SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "SentencePieceProcessor__DecodeIdsBatch" "', argument " "3"" of type '" "int""'");
+ }
+ arg3 = static_cast< int >(val3);
{
try {
- result = sentencepiece_SentencePieceProcessor_LoadFromFile(arg1,arg2);
+ result = sentencepiece_SentencePieceProcessor__DecodeIdsBatch((sentencepiece::SentencePieceProcessor const *)arg1,(std::vector< std::vector< int > > const &)*arg2,arg3);
ReleaseResultObject(resultobj);
}
catch (const sentencepiece::util::Status &status) {
}
}
{
- if (!(&result)->ok()) {
- SWIG_exception(ToSwigError((&result)->code()), (&result)->ToString().c_str());
+ PyObject *input_type = resultobj;
+ resultobj = PyList_New((&result)->size());
+ for (size_t i = 0; i < (&result)->size(); ++i) {
+ PyList_SetItem(resultobj, i, MakePyOutputString(result[i], input_type));
}
- resultobj = SWIG_From_bool((&result)->ok());
+ }
+ {
+ delete arg2;
}
return resultobj;
fail:
+ {
+ delete arg2;
+ }
return NULL;
}
-SWIGINTERN PyObject *_wrap_SentencePieceProcessor_DecodeIdsWithCheck(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+SWIGINTERN PyObject *_wrap_SentencePieceProcessor__DecodeIdsAsSerializedProtoBatch(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
PyObject *resultobj = 0;
sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ;
- std::vector< int > *arg2 = 0 ;
+ std::vector< std::vector< int > > *arg2 = 0 ;
+ int arg3 ;
void *argp1 = 0 ;
int res1 = 0 ;
- PyObject *swig_obj[2] ;
- std::string result;
+ int val3 ;
+ int ecode3 = 0 ;
+ PyObject *swig_obj[3] ;
+ BytesArray result;
- if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor_DecodeIdsWithCheck", 2, 2, swig_obj)) SWIG_fail;
+ if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor__DecodeIdsAsSerializedProtoBatch", 3, 3, swig_obj)) SWIG_fail;
res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 );
if (!SWIG_IsOK(res1)) {
- SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor_DecodeIdsWithCheck" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'");
+ SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor__DecodeIdsAsSerializedProtoBatch" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'");
}
arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1);
{
- std::vector<int> *out = nullptr;
+ std::vector<std::vector<int>> *out = nullptr;
if (PyList_Check(swig_obj[1])) {
const size_t size = PyList_Size(swig_obj[1]);
- out = new std::vector<int>(size);
+ out = new std::vector<std::vector<int>>(size);
for (size_t i = 0; i < size; ++i) {
PyObject *o = PyList_GetItem(swig_obj[1], i);
- if (PyInt_Check(o)) {
- (*out)[i] = static_cast<int>(PyInt_AsLong(o));
+ if (PyList_Check(o)) {
+ const size_t size2 = PyList_Size(o);
+ (*out)[i].resize(size2);
+ for (size_t j = 0; j < size2; ++j) {
+ PyObject *o2 = PyList_GetItem(o, j);
+ if (PyInt_Check(o2)) {
+ (*out)[i][j] = static_cast<int>(PyInt_AsLong(o2));
+ } else {
+ PyErr_SetString(PyExc_TypeError, "list must contain strings");
+ SWIG_fail;
+ }
+ }
} else {
- PyErr_SetString(PyExc_TypeError,"list must contain integers");
+ PyErr_SetString(PyExc_TypeError, "not a list");
SWIG_fail;
}
}
}
arg2 = out;
}
+ ecode3 = SWIG_AsVal_int(swig_obj[2], &val3);
+ if (!SWIG_IsOK(ecode3)) {
+ SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "SentencePieceProcessor__DecodeIdsAsSerializedProtoBatch" "', argument " "3"" of type '" "int""'");
+ }
+ arg3 = static_cast< int >(val3);
{
try {
- result = sentencepiece_SentencePieceProcessor_DecodeIdsWithCheck((sentencepiece::SentencePieceProcessor const *)arg1,(std::vector< int > const &)*arg2);
+ result = sentencepiece_SentencePieceProcessor__DecodeIdsAsSerializedProtoBatch((sentencepiece::SentencePieceProcessor const *)arg1,(std::vector< std::vector< int > > const &)*arg2,arg3);
ReleaseResultObject(resultobj);
}
catch (const sentencepiece::util::Status &status) {
}
}
{
- PyObject *input_type = resultobj;
- resultobj = MakePyOutputString(result, input_type);
+ resultobj = PyList_New((&result)->size());
+ for (size_t i = 0; i < (&result)->size(); ++i) {
+ PyList_SetItem(resultobj, i, MakePyOutputBytes(result[i]));
+ }
}
{
delete arg2;
}
-SWIGINTERN PyObject *_wrap_SentencePieceProcessor_DecodeIdsAsSerializedProtoWithCheck(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+SWIGINTERN PyObject *_wrap_SentencePieceProcessor__DecodePiecesBatch(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
PyObject *resultobj = 0;
sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ;
- std::vector< int > *arg2 = 0 ;
+ std::vector< std::vector< std::string > > *arg2 = 0 ;
+ int arg3 ;
void *argp1 = 0 ;
int res1 = 0 ;
- PyObject *swig_obj[2] ;
- sentencepiece::util::bytes result;
+ int val3 ;
+ int ecode3 = 0 ;
+ PyObject *swig_obj[3] ;
+ std::vector< std::string > result;
- if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor_DecodeIdsAsSerializedProtoWithCheck", 2, 2, swig_obj)) SWIG_fail;
+ if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor__DecodePiecesBatch", 3, 3, swig_obj)) SWIG_fail;
res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 );
if (!SWIG_IsOK(res1)) {
- SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor_DecodeIdsAsSerializedProtoWithCheck" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'");
+ SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor__DecodePiecesBatch" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'");
}
arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1);
{
- std::vector<int> *out = nullptr;
+ std::vector<std::vector<std::string>> *out = nullptr;
if (PyList_Check(swig_obj[1])) {
const size_t size = PyList_Size(swig_obj[1]);
- out = new std::vector<int>(size);
+ out = new std::vector<std::vector<std::string>>(size);
for (size_t i = 0; i < size; ++i) {
PyObject *o = PyList_GetItem(swig_obj[1], i);
- if (PyInt_Check(o)) {
- (*out)[i] = static_cast<int>(PyInt_AsLong(o));
+ if (PyList_Check(o)) {
+ const size_t size2 = PyList_Size(o);
+ (*out)[i].resize(size2);
+ for (size_t j = 0; j < size2; ++j) {
+ const PyInputString ustring(PyList_GetItem(o, j));
+ if (ustring.IsAvalable()) {
+ (*out)[i][j].assign(ustring.data(), ustring.size());
+ } else {
+ PyErr_SetString(PyExc_TypeError,"list must contain integers");
+ SWIG_fail;
+ }
+ resultobj = ustring.input_type();
+ }
} else {
- PyErr_SetString(PyExc_TypeError,"list must contain integers");
+ PyErr_SetString(PyExc_TypeError,"not a list");
SWIG_fail;
}
}
}
arg2 = out;
}
+ ecode3 = SWIG_AsVal_int(swig_obj[2], &val3);
+ if (!SWIG_IsOK(ecode3)) {
+ SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "SentencePieceProcessor__DecodePiecesBatch" "', argument " "3"" of type '" "int""'");
+ }
+ arg3 = static_cast< int >(val3);
{
try {
- result = sentencepiece_SentencePieceProcessor_DecodeIdsAsSerializedProtoWithCheck((sentencepiece::SentencePieceProcessor const *)arg1,(std::vector< int > const &)*arg2);
+ result = sentencepiece_SentencePieceProcessor__DecodePiecesBatch((sentencepiece::SentencePieceProcessor const *)arg1,(std::vector< std::vector< std::string > > const &)*arg2,arg3);
ReleaseResultObject(resultobj);
}
catch (const sentencepiece::util::Status &status) {
}
}
{
- resultobj = MakePyOutputBytes(result);
+ PyObject *input_type = resultobj;
+ resultobj = PyList_New((&result)->size());
+ for (size_t i = 0; i < (&result)->size(); ++i) {
+ PyList_SetItem(resultobj, i, MakePyOutputString(result[i], input_type));
+ }
}
{
delete arg2;
}
-SWIGINTERN PyObject *_wrap_SentencePieceProcessor__EncodeAsIds(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+SWIGINTERN PyObject *_wrap_SentencePieceProcessor__DecodePiecesAsSerializedProtoBatch(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
PyObject *resultobj = 0;
sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ;
- absl::string_view arg2 ;
- bool arg3 ;
- int arg4 ;
- float arg5 ;
- bool arg6 ;
- bool arg7 ;
- bool arg8 ;
+ std::vector< std::vector< std::string > > *arg2 = 0 ;
+ int arg3 ;
void *argp1 = 0 ;
int res1 = 0 ;
- bool val3 ;
+ int val3 ;
int ecode3 = 0 ;
- int val4 ;
- int ecode4 = 0 ;
- float val5 ;
- int ecode5 = 0 ;
- bool val6 ;
- int ecode6 = 0 ;
- bool val7 ;
- int ecode7 = 0 ;
- bool val8 ;
- int ecode8 = 0 ;
- PyObject *swig_obj[8] ;
- std::vector< int > result;
+ PyObject *swig_obj[3] ;
+ BytesArray result;
- if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor__EncodeAsIds", 8, 8, swig_obj)) SWIG_fail;
+ if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor__DecodePiecesAsSerializedProtoBatch", 3, 3, swig_obj)) SWIG_fail;
res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 );
if (!SWIG_IsOK(res1)) {
- SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor__EncodeAsIds" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor *""'");
- }
- arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1);
- {
- const PyInputString ustring(swig_obj[1]);
- if (!ustring.IsAvalable()) {
- PyErr_SetString(PyExc_TypeError, "not a string");
- SWIG_fail;
- }
- resultobj = ustring.input_type();
- arg2 = absl::string_view(ustring.data(), ustring.size());
+ SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor__DecodePiecesAsSerializedProtoBatch" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'");
}
- ecode3 = SWIG_AsVal_bool(swig_obj[2], &val3);
- if (!SWIG_IsOK(ecode3)) {
- SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "SentencePieceProcessor__EncodeAsIds" "', argument " "3"" of type '" "bool""'");
- }
- arg3 = static_cast< bool >(val3);
- ecode4 = SWIG_AsVal_int(swig_obj[3], &val4);
- if (!SWIG_IsOK(ecode4)) {
- SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "SentencePieceProcessor__EncodeAsIds" "', argument " "4"" of type '" "int""'");
- }
- arg4 = static_cast< int >(val4);
- ecode5 = SWIG_AsVal_float(swig_obj[4], &val5);
- if (!SWIG_IsOK(ecode5)) {
- SWIG_exception_fail(SWIG_ArgError(ecode5), "in method '" "SentencePieceProcessor__EncodeAsIds" "', argument " "5"" of type '" "float""'");
- }
- arg5 = static_cast< float >(val5);
- ecode6 = SWIG_AsVal_bool(swig_obj[5], &val6);
- if (!SWIG_IsOK(ecode6)) {
- SWIG_exception_fail(SWIG_ArgError(ecode6), "in method '" "SentencePieceProcessor__EncodeAsIds" "', argument " "6"" of type '" "bool""'");
- }
- arg6 = static_cast< bool >(val6);
- ecode7 = SWIG_AsVal_bool(swig_obj[6], &val7);
- if (!SWIG_IsOK(ecode7)) {
- SWIG_exception_fail(SWIG_ArgError(ecode7), "in method '" "SentencePieceProcessor__EncodeAsIds" "', argument " "7"" of type '" "bool""'");
- }
- arg7 = static_cast< bool >(val7);
- ecode8 = SWIG_AsVal_bool(swig_obj[7], &val8);
- if (!SWIG_IsOK(ecode8)) {
- SWIG_exception_fail(SWIG_ArgError(ecode8), "in method '" "SentencePieceProcessor__EncodeAsIds" "', argument " "8"" of type '" "bool""'");
+ arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1);
+ {
+ std::vector<std::vector<std::string>> *out = nullptr;
+ if (PyList_Check(swig_obj[1])) {
+ const size_t size = PyList_Size(swig_obj[1]);
+ out = new std::vector<std::vector<std::string>>(size);
+ for (size_t i = 0; i < size; ++i) {
+ PyObject *o = PyList_GetItem(swig_obj[1], i);
+ if (PyList_Check(o)) {
+ const size_t size2 = PyList_Size(o);
+ (*out)[i].resize(size2);
+ for (size_t j = 0; j < size2; ++j) {
+ const PyInputString ustring(PyList_GetItem(o, j));
+ if (ustring.IsAvalable()) {
+ (*out)[i][j].assign(ustring.data(), ustring.size());
+ } else {
+ PyErr_SetString(PyExc_TypeError,"list must contain integers");
+ SWIG_fail;
+ }
+ resultobj = ustring.input_type();
+ }
+ } else {
+ PyErr_SetString(PyExc_TypeError,"not a list");
+ SWIG_fail;
+ }
+ }
+ } else {
+ PyErr_SetString(PyExc_TypeError,"not a list");
+ SWIG_fail;
+ }
+ arg2 = out;
+ }
+ ecode3 = SWIG_AsVal_int(swig_obj[2], &val3);
+ if (!SWIG_IsOK(ecode3)) {
+ SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "SentencePieceProcessor__DecodePiecesAsSerializedProtoBatch" "', argument " "3"" of type '" "int""'");
}
- arg8 = static_cast< bool >(val8);
+ arg3 = static_cast< int >(val3);
{
try {
- result = sentencepiece_SentencePieceProcessor__EncodeAsIds(arg1,arg2,arg3,arg4,arg5,arg6,arg7,arg8);
+ result = sentencepiece_SentencePieceProcessor__DecodePiecesAsSerializedProtoBatch((sentencepiece::SentencePieceProcessor const *)arg1,(std::vector< std::vector< std::string > > const &)*arg2,arg3);
ReleaseResultObject(resultobj);
}
catch (const sentencepiece::util::Status &status) {
{
resultobj = PyList_New((&result)->size());
for (size_t i = 0; i < (&result)->size(); ++i) {
- PyList_SetItem(resultobj, i, PyInt_FromLong(static_cast<long>(result[i])));
+ PyList_SetItem(resultobj, i, MakePyOutputBytes(result[i]));
}
}
+ {
+ delete arg2;
+ }
return resultobj;
fail:
+ {
+ delete arg2;
+ }
return NULL;
}
-SWIGINTERN PyObject *_wrap_SentencePieceProcessor__EncodeAsPieces(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+SWIGINTERN PyObject *_wrap_SentencePieceProcessor__NBestEncodeAsIds(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
PyObject *resultobj = 0;
sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ;
absl::string_view arg2 ;
- bool arg3 ;
- int arg4 ;
- float arg5 ;
+ int arg3 ;
+ bool arg4 ;
+ bool arg5 ;
bool arg6 ;
bool arg7 ;
- bool arg8 ;
- bool arg9 ;
void *argp1 = 0 ;
int res1 = 0 ;
- bool val3 ;
+ int val3 ;
int ecode3 = 0 ;
- int val4 ;
+ bool val4 ;
int ecode4 = 0 ;
- float val5 ;
+ bool val5 ;
int ecode5 = 0 ;
bool val6 ;
int ecode6 = 0 ;
bool val7 ;
int ecode7 = 0 ;
- bool val8 ;
- int ecode8 = 0 ;
- bool val9 ;
- int ecode9 = 0 ;
- PyObject *swig_obj[9] ;
- std::vector< std::string > result;
+ PyObject *swig_obj[7] ;
+ std::vector< std::vector< int > > result;
- if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor__EncodeAsPieces", 9, 9, swig_obj)) SWIG_fail;
+ if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor__NBestEncodeAsIds", 7, 7, swig_obj)) SWIG_fail;
res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 );
if (!SWIG_IsOK(res1)) {
- SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor__EncodeAsPieces" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor *""'");
+ SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor__NBestEncodeAsIds" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'");
}
arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1);
{
resultobj = ustring.input_type();
arg2 = absl::string_view(ustring.data(), ustring.size());
}
- ecode3 = SWIG_AsVal_bool(swig_obj[2], &val3);
+ ecode3 = SWIG_AsVal_int(swig_obj[2], &val3);
if (!SWIG_IsOK(ecode3)) {
- SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "SentencePieceProcessor__EncodeAsPieces" "', argument " "3"" of type '" "bool""'");
+ SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "SentencePieceProcessor__NBestEncodeAsIds" "', argument " "3"" of type '" "int""'");
}
- arg3 = static_cast< bool >(val3);
- ecode4 = SWIG_AsVal_int(swig_obj[3], &val4);
+ arg3 = static_cast< int >(val3);
+ ecode4 = SWIG_AsVal_bool(swig_obj[3], &val4);
if (!SWIG_IsOK(ecode4)) {
- SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "SentencePieceProcessor__EncodeAsPieces" "', argument " "4"" of type '" "int""'");
+ SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "SentencePieceProcessor__NBestEncodeAsIds" "', argument " "4"" of type '" "bool""'");
}
- arg4 = static_cast< int >(val4);
- ecode5 = SWIG_AsVal_float(swig_obj[4], &val5);
+ arg4 = static_cast< bool >(val4);
+ ecode5 = SWIG_AsVal_bool(swig_obj[4], &val5);
if (!SWIG_IsOK(ecode5)) {
- SWIG_exception_fail(SWIG_ArgError(ecode5), "in method '" "SentencePieceProcessor__EncodeAsPieces" "', argument " "5"" of type '" "float""'");
+ SWIG_exception_fail(SWIG_ArgError(ecode5), "in method '" "SentencePieceProcessor__NBestEncodeAsIds" "', argument " "5"" of type '" "bool""'");
}
- arg5 = static_cast< float >(val5);
+ arg5 = static_cast< bool >(val5);
ecode6 = SWIG_AsVal_bool(swig_obj[5], &val6);
if (!SWIG_IsOK(ecode6)) {
- SWIG_exception_fail(SWIG_ArgError(ecode6), "in method '" "SentencePieceProcessor__EncodeAsPieces" "', argument " "6"" of type '" "bool""'");
+ SWIG_exception_fail(SWIG_ArgError(ecode6), "in method '" "SentencePieceProcessor__NBestEncodeAsIds" "', argument " "6"" of type '" "bool""'");
}
arg6 = static_cast< bool >(val6);
ecode7 = SWIG_AsVal_bool(swig_obj[6], &val7);
if (!SWIG_IsOK(ecode7)) {
- SWIG_exception_fail(SWIG_ArgError(ecode7), "in method '" "SentencePieceProcessor__EncodeAsPieces" "', argument " "7"" of type '" "bool""'");
+ SWIG_exception_fail(SWIG_ArgError(ecode7), "in method '" "SentencePieceProcessor__NBestEncodeAsIds" "', argument " "7"" of type '" "bool""'");
}
arg7 = static_cast< bool >(val7);
- ecode8 = SWIG_AsVal_bool(swig_obj[7], &val8);
- if (!SWIG_IsOK(ecode8)) {
- SWIG_exception_fail(SWIG_ArgError(ecode8), "in method '" "SentencePieceProcessor__EncodeAsPieces" "', argument " "8"" of type '" "bool""'");
- }
- arg8 = static_cast< bool >(val8);
- ecode9 = SWIG_AsVal_bool(swig_obj[8], &val9);
- if (!SWIG_IsOK(ecode9)) {
- SWIG_exception_fail(SWIG_ArgError(ecode9), "in method '" "SentencePieceProcessor__EncodeAsPieces" "', argument " "9"" of type '" "bool""'");
- }
- arg9 = static_cast< bool >(val9);
{
try {
- result = sentencepiece_SentencePieceProcessor__EncodeAsPieces(arg1,arg2,arg3,arg4,arg5,arg6,arg7,arg8,arg9);
+ result = sentencepiece_SentencePieceProcessor__NBestEncodeAsIds((sentencepiece::SentencePieceProcessor const *)arg1,arg2,arg3,arg4,arg5,arg6,arg7);
ReleaseResultObject(resultobj);
}
catch (const sentencepiece::util::Status &status) {
}
}
{
- PyObject *input_type = resultobj;
resultobj = PyList_New((&result)->size());
for (size_t i = 0; i < (&result)->size(); ++i) {
- PyList_SetItem(resultobj, i, MakePyOutputString(result[i], input_type));
+ PyObject *obj = PyList_New(result[i].size());
+ for (size_t j = 0; j < result[i].size(); ++j) {
+ PyList_SetItem(obj, j, PyInt_FromLong(static_cast<long>(result[i][j])));
+ }
+ PyList_SetItem(resultobj, i, obj);
}
}
return resultobj;
}
-SWIGINTERN PyObject *_wrap_SentencePieceProcessor__NBestEncodeAsIds(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+SWIGINTERN PyObject *_wrap_SentencePieceProcessor__NBestEncodeAsPieces(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
PyObject *resultobj = 0;
sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ;
absl::string_view arg2 ;
bool arg4 ;
bool arg5 ;
bool arg6 ;
+ bool arg7 ;
void *argp1 = 0 ;
int res1 = 0 ;
int val3 ;
int ecode5 = 0 ;
bool val6 ;
int ecode6 = 0 ;
- PyObject *swig_obj[6] ;
- std::vector< std::vector< int > > result;
+ bool val7 ;
+ int ecode7 = 0 ;
+ PyObject *swig_obj[7] ;
+ std::vector< std::vector< std::string > > result;
- if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor__NBestEncodeAsIds", 6, 6, swig_obj)) SWIG_fail;
+ if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor__NBestEncodeAsPieces", 7, 7, swig_obj)) SWIG_fail;
res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 );
if (!SWIG_IsOK(res1)) {
- SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor__NBestEncodeAsIds" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor *""'");
+ SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor__NBestEncodeAsPieces" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'");
}
arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1);
{
}
ecode3 = SWIG_AsVal_int(swig_obj[2], &val3);
if (!SWIG_IsOK(ecode3)) {
- SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "SentencePieceProcessor__NBestEncodeAsIds" "', argument " "3"" of type '" "int""'");
+ SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "SentencePieceProcessor__NBestEncodeAsPieces" "', argument " "3"" of type '" "int""'");
}
arg3 = static_cast< int >(val3);
ecode4 = SWIG_AsVal_bool(swig_obj[3], &val4);
if (!SWIG_IsOK(ecode4)) {
- SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "SentencePieceProcessor__NBestEncodeAsIds" "', argument " "4"" of type '" "bool""'");
+ SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "SentencePieceProcessor__NBestEncodeAsPieces" "', argument " "4"" of type '" "bool""'");
}
arg4 = static_cast< bool >(val4);
ecode5 = SWIG_AsVal_bool(swig_obj[4], &val5);
if (!SWIG_IsOK(ecode5)) {
- SWIG_exception_fail(SWIG_ArgError(ecode5), "in method '" "SentencePieceProcessor__NBestEncodeAsIds" "', argument " "5"" of type '" "bool""'");
+ SWIG_exception_fail(SWIG_ArgError(ecode5), "in method '" "SentencePieceProcessor__NBestEncodeAsPieces" "', argument " "5"" of type '" "bool""'");
}
arg5 = static_cast< bool >(val5);
ecode6 = SWIG_AsVal_bool(swig_obj[5], &val6);
if (!SWIG_IsOK(ecode6)) {
- SWIG_exception_fail(SWIG_ArgError(ecode6), "in method '" "SentencePieceProcessor__NBestEncodeAsIds" "', argument " "6"" of type '" "bool""'");
+ SWIG_exception_fail(SWIG_ArgError(ecode6), "in method '" "SentencePieceProcessor__NBestEncodeAsPieces" "', argument " "6"" of type '" "bool""'");
}
arg6 = static_cast< bool >(val6);
+ ecode7 = SWIG_AsVal_bool(swig_obj[6], &val7);
+ if (!SWIG_IsOK(ecode7)) {
+ SWIG_exception_fail(SWIG_ArgError(ecode7), "in method '" "SentencePieceProcessor__NBestEncodeAsPieces" "', argument " "7"" of type '" "bool""'");
+ }
+ arg7 = static_cast< bool >(val7);
{
try {
- result = sentencepiece_SentencePieceProcessor__NBestEncodeAsIds(arg1,arg2,arg3,arg4,arg5,arg6);
+ result = sentencepiece_SentencePieceProcessor__NBestEncodeAsPieces((sentencepiece::SentencePieceProcessor const *)arg1,arg2,arg3,arg4,arg5,arg6,arg7);
ReleaseResultObject(resultobj);
}
catch (const sentencepiece::util::Status &status) {
}
}
{
+ PyObject *input_type = resultobj;
resultobj = PyList_New((&result)->size());
for (size_t i = 0; i < (&result)->size(); ++i) {
PyObject *obj = PyList_New(result[i].size());
for (size_t j = 0; j < result[i].size(); ++j) {
- PyList_SetItem(obj, j, PyInt_FromLong(static_cast<long>(result[i][j])));
+ PyList_SetItem(obj, j, MakePyOutputString(result[i][j], input_type));
}
PyList_SetItem(resultobj, i, obj);
}
}
-SWIGINTERN PyObject *_wrap_SentencePieceProcessor__NBestEncodeAsPieces(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+SWIGINTERN PyObject *_wrap_SentencePieceProcessor__NBestEncodeAsSerializedProto(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
PyObject *resultobj = 0;
sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ;
absl::string_view arg2 ;
bool val7 ;
int ecode7 = 0 ;
PyObject *swig_obj[7] ;
- std::vector< std::vector< std::string > > result;
+ sentencepiece::util::bytes result;
- if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor__NBestEncodeAsPieces", 7, 7, swig_obj)) SWIG_fail;
+ if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor__NBestEncodeAsSerializedProto", 7, 7, swig_obj)) SWIG_fail;
res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 );
if (!SWIG_IsOK(res1)) {
- SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor__NBestEncodeAsPieces" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor *""'");
+ SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor__NBestEncodeAsSerializedProto" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'");
}
arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1);
{
}
ecode3 = SWIG_AsVal_int(swig_obj[2], &val3);
if (!SWIG_IsOK(ecode3)) {
- SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "SentencePieceProcessor__NBestEncodeAsPieces" "', argument " "3"" of type '" "int""'");
+ SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "SentencePieceProcessor__NBestEncodeAsSerializedProto" "', argument " "3"" of type '" "int""'");
}
arg3 = static_cast< int >(val3);
ecode4 = SWIG_AsVal_bool(swig_obj[3], &val4);
if (!SWIG_IsOK(ecode4)) {
- SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "SentencePieceProcessor__NBestEncodeAsPieces" "', argument " "4"" of type '" "bool""'");
+ SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "SentencePieceProcessor__NBestEncodeAsSerializedProto" "', argument " "4"" of type '" "bool""'");
}
arg4 = static_cast< bool >(val4);
ecode5 = SWIG_AsVal_bool(swig_obj[4], &val5);
if (!SWIG_IsOK(ecode5)) {
- SWIG_exception_fail(SWIG_ArgError(ecode5), "in method '" "SentencePieceProcessor__NBestEncodeAsPieces" "', argument " "5"" of type '" "bool""'");
+ SWIG_exception_fail(SWIG_ArgError(ecode5), "in method '" "SentencePieceProcessor__NBestEncodeAsSerializedProto" "', argument " "5"" of type '" "bool""'");
}
arg5 = static_cast< bool >(val5);
ecode6 = SWIG_AsVal_bool(swig_obj[5], &val6);
if (!SWIG_IsOK(ecode6)) {
- SWIG_exception_fail(SWIG_ArgError(ecode6), "in method '" "SentencePieceProcessor__NBestEncodeAsPieces" "', argument " "6"" of type '" "bool""'");
+ SWIG_exception_fail(SWIG_ArgError(ecode6), "in method '" "SentencePieceProcessor__NBestEncodeAsSerializedProto" "', argument " "6"" of type '" "bool""'");
}
arg6 = static_cast< bool >(val6);
ecode7 = SWIG_AsVal_bool(swig_obj[6], &val7);
if (!SWIG_IsOK(ecode7)) {
- SWIG_exception_fail(SWIG_ArgError(ecode7), "in method '" "SentencePieceProcessor__NBestEncodeAsPieces" "', argument " "7"" of type '" "bool""'");
+ SWIG_exception_fail(SWIG_ArgError(ecode7), "in method '" "SentencePieceProcessor__NBestEncodeAsSerializedProto" "', argument " "7"" of type '" "bool""'");
}
arg7 = static_cast< bool >(val7);
{
try {
- result = sentencepiece_SentencePieceProcessor__NBestEncodeAsPieces(arg1,arg2,arg3,arg4,arg5,arg6,arg7);
+ result = sentencepiece_SentencePieceProcessor__NBestEncodeAsSerializedProto((sentencepiece::SentencePieceProcessor const *)arg1,arg2,arg3,arg4,arg5,arg6,arg7);
ReleaseResultObject(resultobj);
}
catch (const sentencepiece::util::Status &status) {
}
}
{
- PyObject *input_type = resultobj;
- resultobj = PyList_New((&result)->size());
- for (size_t i = 0; i < (&result)->size(); ++i) {
- PyObject *obj = PyList_New(result[i].size());
- for (size_t j = 0; j < result[i].size(); ++j) {
- PyList_SetItem(obj, j, MakePyOutputString(result[i][j], input_type));
- }
- PyList_SetItem(resultobj, i, obj);
- }
+ resultobj = MakePyOutputBytes(result);
}
return resultobj;
fail:
bool arg7 ;
bool arg8 ;
bool arg9 ;
+ bool arg10 ;
void *argp1 = 0 ;
int res1 = 0 ;
int val3 ;
int ecode8 = 0 ;
bool val9 ;
int ecode9 = 0 ;
- PyObject *swig_obj[9] ;
+ bool val10 ;
+ int ecode10 = 0 ;
+ PyObject *swig_obj[10] ;
std::vector< std::pair< std::vector< int >,float > > result;
- if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor__SampleEncodeAndScoreAsIds", 9, 9, swig_obj)) SWIG_fail;
+ if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor__SampleEncodeAndScoreAsIds", 10, 10, swig_obj)) SWIG_fail;
res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 );
if (!SWIG_IsOK(res1)) {
- SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor__SampleEncodeAndScoreAsIds" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor *""'");
+ SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor__SampleEncodeAndScoreAsIds" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'");
}
arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1);
{
SWIG_exception_fail(SWIG_ArgError(ecode9), "in method '" "SentencePieceProcessor__SampleEncodeAndScoreAsIds" "', argument " "9"" of type '" "bool""'");
}
arg9 = static_cast< bool >(val9);
+ ecode10 = SWIG_AsVal_bool(swig_obj[9], &val10);
+ if (!SWIG_IsOK(ecode10)) {
+ SWIG_exception_fail(SWIG_ArgError(ecode10), "in method '" "SentencePieceProcessor__SampleEncodeAndScoreAsIds" "', argument " "10"" of type '" "bool""'");
+ }
+ arg10 = static_cast< bool >(val10);
{
try {
- result = sentencepiece_SentencePieceProcessor__SampleEncodeAndScoreAsIds(arg1,arg2,arg3,arg4,arg5,arg6,arg7,arg8,arg9);
+ result = sentencepiece_SentencePieceProcessor__SampleEncodeAndScoreAsIds((sentencepiece::SentencePieceProcessor const *)arg1,arg2,arg3,arg4,arg5,arg6,arg7,arg8,arg9,arg10);
ReleaseResultObject(resultobj);
}
catch (const sentencepiece::util::Status &status) {
if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor__SampleEncodeAndScoreAsPieces", 10, 10, swig_obj)) SWIG_fail;
res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 );
if (!SWIG_IsOK(res1)) {
- SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor__SampleEncodeAndScoreAsPieces" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor *""'");
+ SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor__SampleEncodeAndScoreAsPieces" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'");
}
arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1);
{
arg10 = static_cast< bool >(val10);
{
try {
- result = sentencepiece_SentencePieceProcessor__SampleEncodeAndScoreAsPieces(arg1,arg2,arg3,arg4,arg5,arg6,arg7,arg8,arg9,arg10);
+ result = sentencepiece_SentencePieceProcessor__SampleEncodeAndScoreAsPieces((sentencepiece::SentencePieceProcessor const *)arg1,arg2,arg3,arg4,arg5,arg6,arg7,arg8,arg9,arg10);
ReleaseResultObject(resultobj);
}
catch (const sentencepiece::util::Status &status) {
}
+SWIGINTERN PyObject *_wrap_SentencePieceProcessor__CalculateEntropy(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+ PyObject *resultobj = 0;
+ sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ;
+ absl::string_view arg2 ;
+ float arg3 ;
+ void *argp1 = 0 ;
+ int res1 = 0 ;
+ float val3 ;
+ int ecode3 = 0 ;
+ PyObject *swig_obj[3] ;
+ float result;
+
+ if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor__CalculateEntropy", 3, 3, swig_obj)) SWIG_fail;
+ res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 );
+ if (!SWIG_IsOK(res1)) {
+ SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor__CalculateEntropy" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor *""'");
+ }
+ arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1);
+ {
+ const PyInputString ustring(swig_obj[1]);
+ if (!ustring.IsAvalable()) {
+ PyErr_SetString(PyExc_TypeError, "not a string");
+ SWIG_fail;
+ }
+ resultobj = ustring.input_type();
+ arg2 = absl::string_view(ustring.data(), ustring.size());
+ }
+ ecode3 = SWIG_AsVal_float(swig_obj[2], &val3);
+ if (!SWIG_IsOK(ecode3)) {
+ SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "SentencePieceProcessor__CalculateEntropy" "', argument " "3"" of type '" "float""'");
+ }
+ arg3 = static_cast< float >(val3);
+ {
+ try {
+ result = (float)sentencepiece_SentencePieceProcessor__CalculateEntropy(arg1,arg2,arg3);
+ ReleaseResultObject(resultobj);
+ }
+ catch (const sentencepiece::util::Status &status) {
+ SWIG_exception(ToSwigError(status.code()), status.ToString().c_str());
+ }
+ }
+ resultobj = SWIG_From_float(static_cast< float >(result));
+ return resultobj;
+fail:
+ return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_SentencePieceProcessor__CalculateEntropyBatch(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+ PyObject *resultobj = 0;
+ sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ;
+ std::vector< absl::string_view > *arg2 = 0 ;
+ float arg3 ;
+ int arg4 ;
+ void *argp1 = 0 ;
+ int res1 = 0 ;
+ float val3 ;
+ int ecode3 = 0 ;
+ int val4 ;
+ int ecode4 = 0 ;
+ PyObject *swig_obj[4] ;
+ std::vector< float > result;
+
+ if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor__CalculateEntropyBatch", 4, 4, swig_obj)) SWIG_fail;
+ res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 );
+ if (!SWIG_IsOK(res1)) {
+ SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor__CalculateEntropyBatch" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor *""'");
+ }
+ arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1);
+ {
+ std::vector<absl::string_view> *out = nullptr;
+ if (PyList_Check(swig_obj[1])) {
+ const size_t size = PyList_Size(swig_obj[1]);
+ out = new std::vector<absl::string_view>(size);
+ for (size_t i = 0; i < size; ++i) {
+ const PyInputString ustring(PyList_GetItem(swig_obj[1], i));
+ if (ustring.IsAvalable()) {
+ (*out)[i] = absl::string_view(ustring.data(), ustring.size());
+ } else {
+ PyErr_SetString(PyExc_TypeError, "list must contain strings");
+ SWIG_fail;
+ }
+ resultobj = ustring.input_type();
+ }
+ } else {
+ PyErr_SetString(PyExc_TypeError, "not a list");
+ SWIG_fail;
+ }
+ arg2 = out;
+ }
+ ecode3 = SWIG_AsVal_float(swig_obj[2], &val3);
+ if (!SWIG_IsOK(ecode3)) {
+ SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "SentencePieceProcessor__CalculateEntropyBatch" "', argument " "3"" of type '" "float""'");
+ }
+ arg3 = static_cast< float >(val3);
+ ecode4 = SWIG_AsVal_int(swig_obj[3], &val4);
+ if (!SWIG_IsOK(ecode4)) {
+ SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "SentencePieceProcessor__CalculateEntropyBatch" "', argument " "4"" of type '" "int""'");
+ }
+ arg4 = static_cast< int >(val4);
+ {
+ try {
+ result = sentencepiece_SentencePieceProcessor__CalculateEntropyBatch(arg1,(std::vector< absl::string_view > const &)*arg2,arg3,arg4);
+ ReleaseResultObject(resultobj);
+ }
+ catch (const sentencepiece::util::Status &status) {
+ SWIG_exception(ToSwigError(status.code()), status.ToString().c_str());
+ }
+ }
+ {
+ resultobj = PyList_New((&result)->size());
+ for (size_t i = 0; i < (&result)->size(); ++i) {
+ PyList_SetItem(resultobj, i, PyFloat_FromDouble(static_cast<double>(result[i])));
+ }
+ }
+ {
+ delete arg2;
+ }
+ return resultobj;
+fail:
+ {
+ delete arg2;
+ }
+ return NULL;
+}
+
+
SWIGINTERN PyObject *SentencePieceProcessor_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
PyObject *obj;
if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL;
{ "SentencePieceProcessor_SetVocabulary", _wrap_SentencePieceProcessor_SetVocabulary, METH_VARARGS, NULL},
{ "SentencePieceProcessor_ResetVocabulary", _wrap_SentencePieceProcessor_ResetVocabulary, METH_O, NULL},
{ "SentencePieceProcessor_LoadVocabulary", _wrap_SentencePieceProcessor_LoadVocabulary, METH_VARARGS, NULL},
- { "SentencePieceProcessor_EncodeAsPieces", _wrap_SentencePieceProcessor_EncodeAsPieces, METH_VARARGS, NULL},
- { "SentencePieceProcessor_EncodeAsIds", _wrap_SentencePieceProcessor_EncodeAsIds, METH_VARARGS, NULL},
- { "SentencePieceProcessor_NBestEncodeAsPieces", _wrap_SentencePieceProcessor_NBestEncodeAsPieces, METH_VARARGS, NULL},
- { "SentencePieceProcessor_NBestEncodeAsIds", _wrap_SentencePieceProcessor_NBestEncodeAsIds, METH_VARARGS, NULL},
- { "SentencePieceProcessor_SampleEncodeAsPieces", _wrap_SentencePieceProcessor_SampleEncodeAsPieces, METH_VARARGS, NULL},
- { "SentencePieceProcessor_SampleEncodeAsIds", _wrap_SentencePieceProcessor_SampleEncodeAsIds, METH_VARARGS, NULL},
{ "SentencePieceProcessor_SampleEncodeAndScoreAsPieces", _wrap_SentencePieceProcessor_SampleEncodeAndScoreAsPieces, METH_VARARGS, NULL},
{ "SentencePieceProcessor_SampleEncodeAndScoreAsIds", _wrap_SentencePieceProcessor_SampleEncodeAndScoreAsIds, METH_VARARGS, NULL},
- { "SentencePieceProcessor_DecodePieces", _wrap_SentencePieceProcessor_DecodePieces, METH_VARARGS, NULL},
{ "SentencePieceProcessor_CalculateEntropy", _wrap_SentencePieceProcessor_CalculateEntropy, METH_VARARGS, NULL},
- { "SentencePieceProcessor_EncodeAsSerializedProto", _wrap_SentencePieceProcessor_EncodeAsSerializedProto, METH_VARARGS, NULL},
- { "SentencePieceProcessor_SampleEncodeAsSerializedProto", _wrap_SentencePieceProcessor_SampleEncodeAsSerializedProto, METH_VARARGS, NULL},
- { "SentencePieceProcessor_NBestEncodeAsSerializedProto", _wrap_SentencePieceProcessor_NBestEncodeAsSerializedProto, METH_VARARGS, NULL},
- { "SentencePieceProcessor_DecodePiecesAsSerializedProto", _wrap_SentencePieceProcessor_DecodePiecesAsSerializedProto, METH_VARARGS, NULL},
{ "SentencePieceProcessor_GetPieceSize", _wrap_SentencePieceProcessor_GetPieceSize, METH_O, NULL},
{ "SentencePieceProcessor_PieceToId", _wrap_SentencePieceProcessor_PieceToId, METH_VARARGS, NULL},
{ "SentencePieceProcessor_IdToPiece", _wrap_SentencePieceProcessor_IdToPiece, METH_VARARGS, NULL},
{ "SentencePieceProcessor_pad_id", _wrap_SentencePieceProcessor_pad_id, METH_O, NULL},
{ "SentencePieceProcessor_serialized_model_proto", _wrap_SentencePieceProcessor_serialized_model_proto, METH_O, NULL},
{ "SentencePieceProcessor_LoadFromFile", _wrap_SentencePieceProcessor_LoadFromFile, METH_VARARGS, NULL},
- { "SentencePieceProcessor_DecodeIdsWithCheck", _wrap_SentencePieceProcessor_DecodeIdsWithCheck, METH_VARARGS, NULL},
- { "SentencePieceProcessor_DecodeIdsAsSerializedProtoWithCheck", _wrap_SentencePieceProcessor_DecodeIdsAsSerializedProtoWithCheck, METH_VARARGS, NULL},
{ "SentencePieceProcessor__EncodeAsIds", _wrap_SentencePieceProcessor__EncodeAsIds, METH_VARARGS, NULL},
{ "SentencePieceProcessor__EncodeAsPieces", _wrap_SentencePieceProcessor__EncodeAsPieces, METH_VARARGS, NULL},
+ { "SentencePieceProcessor__EncodeAsSerializedProto", _wrap_SentencePieceProcessor__EncodeAsSerializedProto, METH_VARARGS, NULL},
+ { "SentencePieceProcessor__EncodeAsIdsBatch", _wrap_SentencePieceProcessor__EncodeAsIdsBatch, METH_VARARGS, NULL},
+ { "SentencePieceProcessor__EncodeAsPiecesBatch", _wrap_SentencePieceProcessor__EncodeAsPiecesBatch, METH_VARARGS, NULL},
+ { "SentencePieceProcessor__EncodeAsSerializedProtoBatch", _wrap_SentencePieceProcessor__EncodeAsSerializedProtoBatch, METH_VARARGS, NULL},
+ { "SentencePieceProcessor__DecodeIds", _wrap_SentencePieceProcessor__DecodeIds, METH_VARARGS, NULL},
+ { "SentencePieceProcessor__DecodePieces", _wrap_SentencePieceProcessor__DecodePieces, METH_VARARGS, NULL},
+ { "SentencePieceProcessor__DecodeIdsAsSerializedProto", _wrap_SentencePieceProcessor__DecodeIdsAsSerializedProto, METH_VARARGS, NULL},
+ { "SentencePieceProcessor__DecodePiecesAsSerializedProto", _wrap_SentencePieceProcessor__DecodePiecesAsSerializedProto, METH_VARARGS, NULL},
+ { "SentencePieceProcessor__DecodeIdsBatch", _wrap_SentencePieceProcessor__DecodeIdsBatch, METH_VARARGS, NULL},
+ { "SentencePieceProcessor__DecodeIdsAsSerializedProtoBatch", _wrap_SentencePieceProcessor__DecodeIdsAsSerializedProtoBatch, METH_VARARGS, NULL},
+ { "SentencePieceProcessor__DecodePiecesBatch", _wrap_SentencePieceProcessor__DecodePiecesBatch, METH_VARARGS, NULL},
+ { "SentencePieceProcessor__DecodePiecesAsSerializedProtoBatch", _wrap_SentencePieceProcessor__DecodePiecesAsSerializedProtoBatch, METH_VARARGS, NULL},
{ "SentencePieceProcessor__NBestEncodeAsIds", _wrap_SentencePieceProcessor__NBestEncodeAsIds, METH_VARARGS, NULL},
{ "SentencePieceProcessor__NBestEncodeAsPieces", _wrap_SentencePieceProcessor__NBestEncodeAsPieces, METH_VARARGS, NULL},
+ { "SentencePieceProcessor__NBestEncodeAsSerializedProto", _wrap_SentencePieceProcessor__NBestEncodeAsSerializedProto, METH_VARARGS, NULL},
{ "SentencePieceProcessor__SampleEncodeAndScoreAsIds", _wrap_SentencePieceProcessor__SampleEncodeAndScoreAsIds, METH_VARARGS, NULL},
{ "SentencePieceProcessor__SampleEncodeAndScoreAsPieces", _wrap_SentencePieceProcessor__SampleEncodeAndScoreAsPieces, METH_VARARGS, NULL},
+ { "SentencePieceProcessor__CalculateEntropy", _wrap_SentencePieceProcessor__CalculateEntropy, METH_VARARGS, NULL},
+ { "SentencePieceProcessor__CalculateEntropyBatch", _wrap_SentencePieceProcessor__CalculateEntropyBatch, METH_VARARGS, NULL},
{ "SentencePieceProcessor_swigregister", SentencePieceProcessor_swigregister, METH_O, NULL},
{ "SentencePieceProcessor_swiginit", SentencePieceProcessor_swiginit, METH_VARARGS, NULL},
{ "SetRandomGeneratorSeed", _wrap_SetRandomGeneratorSeed, METH_O, NULL},
static swig_type_info _swigt__p_sentencepiece__SentencePieceTrainer = {"_p_sentencepiece__SentencePieceTrainer", "sentencepiece::SentencePieceTrainer *", 0, 0, (void*)0, 0};
static swig_type_info _swigt__p_std__string = {"_p_std__string", "sentencepiece::util::bytes *|std::string *", 0, 0, (void*)0, 0};
static swig_type_info _swigt__p_std__unordered_mapT_std__string_std__string_t = {"_p_std__unordered_mapT_std__string_std__string_t", "std::unordered_map< std::string,std::string > *", 0, 0, (void*)0, 0};
+static swig_type_info _swigt__p_std__vectorT_absl__string_view_t = {"_p_std__vectorT_absl__string_view_t", "std::vector< absl::string_view > *", 0, 0, (void*)0, 0};
static swig_type_info _swigt__p_std__vectorT_int_t = {"_p_std__vectorT_int_t", "std::vector< int > *", 0, 0, (void*)0, 0};
static swig_type_info _swigt__p_std__vectorT_std__string_t = {"_p_std__vectorT_std__string_t", "std::vector< std::string > *", 0, 0, (void*)0, 0};
+static swig_type_info _swigt__p_std__vectorT_std__vectorT_int_t_t = {"_p_std__vectorT_std__vectorT_int_t_t", "std::vector< std::vector< int > > *", 0, 0, (void*)0, 0};
+static swig_type_info _swigt__p_std__vectorT_std__vectorT_std__string_t_t = {"_p_std__vectorT_std__vectorT_std__string_t_t", "std::vector< std::vector< std::string > > *", 0, 0, (void*)0, 0};
static swig_type_info *swig_type_initial[] = {
&_swigt__p_char,
&_swigt__p_sentencepiece__SentencePieceTrainer,
&_swigt__p_std__string,
&_swigt__p_std__unordered_mapT_std__string_std__string_t,
+ &_swigt__p_std__vectorT_absl__string_view_t,
&_swigt__p_std__vectorT_int_t,
&_swigt__p_std__vectorT_std__string_t,
+ &_swigt__p_std__vectorT_std__vectorT_int_t_t,
+ &_swigt__p_std__vectorT_std__vectorT_std__string_t_t,
};
static swig_cast_info _swigc__p_char[] = { {&_swigt__p_char, 0, 0, 0},{0, 0, 0, 0}};
static swig_cast_info _swigc__p_sentencepiece__SentencePieceTrainer[] = { {&_swigt__p_sentencepiece__SentencePieceTrainer, 0, 0, 0},{0, 0, 0, 0}};
static swig_cast_info _swigc__p_std__string[] = { {&_swigt__p_std__string, 0, 0, 0},{0, 0, 0, 0}};
static swig_cast_info _swigc__p_std__unordered_mapT_std__string_std__string_t[] = { {&_swigt__p_std__unordered_mapT_std__string_std__string_t, 0, 0, 0},{0, 0, 0, 0}};
+static swig_cast_info _swigc__p_std__vectorT_absl__string_view_t[] = { {&_swigt__p_std__vectorT_absl__string_view_t, 0, 0, 0},{0, 0, 0, 0}};
static swig_cast_info _swigc__p_std__vectorT_int_t[] = { {&_swigt__p_std__vectorT_int_t, 0, 0, 0},{0, 0, 0, 0}};
static swig_cast_info _swigc__p_std__vectorT_std__string_t[] = { {&_swigt__p_std__vectorT_std__string_t, 0, 0, 0},{0, 0, 0, 0}};
+static swig_cast_info _swigc__p_std__vectorT_std__vectorT_int_t_t[] = { {&_swigt__p_std__vectorT_std__vectorT_int_t_t, 0, 0, 0},{0, 0, 0, 0}};
+static swig_cast_info _swigc__p_std__vectorT_std__vectorT_std__string_t_t[] = { {&_swigt__p_std__vectorT_std__vectorT_std__string_t_t, 0, 0, 0},{0, 0, 0, 0}};
static swig_cast_info *swig_cast_initial[] = {
_swigc__p_char,
_swigc__p_sentencepiece__SentencePieceTrainer,
_swigc__p_std__string,
_swigc__p_std__unordered_mapT_std__string_std__string_t,
+ _swigc__p_std__vectorT_absl__string_view_t,
_swigc__p_std__vectorT_int_t,
_swigc__p_std__vectorT_std__string_t,
+ _swigc__p_std__vectorT_std__vectorT_int_t_t,
+ _swigc__p_std__vectorT_std__vectorT_std__string_t_t,
};
projects / sentencepiece.git / commitdiff